标签：etcd

1. etcd安装

rpm -ivh etcd-3.2.15-1.el7.x86_64.rpm
systemctl daemon-reload
systemctl enable etcd
systemctl start etcd
export ETCDCTL_API=3
systemctl status etcd

hosts如下

192.168.0.100 etcd01
192.168.0.101 etcd02
192.168.0.102 etcd03

2. etcd配置

etcd02配置如下，详细见kubernetes1.9版本集群配置向导

# egrep -v "^$|^#" /etc/etcd/etcd.conf 
ETCD_DATA_DIR="/var/lib/etcd/"
ETCD_LISTEN_PEER_URLS="https://192.168.0.101:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.0.101:2379,http://127.0.0.1:2379"
ETCD_NAME="etcd02"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.0.101:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.0.101:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.0.100:2380,etcd02=https://192.168.0.101:2380,etcd03=https://192.168.0.102:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="existing"
ETCD_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/etcd-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/ca.pem"
ETCD_PEER_AUTO_TLS="true"

3. 故障报错

3个节点做集群，直接关机后，etcd02故障，报错：

etcd: advertise client URLs = https://192.168.0.101:2379
etcd: read wal error (wal: crc mismatch) and cannot be repaired
systemd: etcd.service: main process exited, code=exited, status=1/FAILURE

wal的cec校验出错，谷歌了一下，没什么结果，于是移除这个etcd，再恢复
在正常的etcd节点移除

# etcdctl member list
1ce6d6d01109192, started, etcd03, https://192.168.0.102:2380, https://192.168.0.102:2379
9b534175b46ea789, started, etcd01, https://192.168.0.100:2380, https://192.168.0.100:2379
ac2f188e97f50eb7, started, etcd02, https://192.168.0.101:2380, https://192.168.0.101:2379
# etcdctl member remove ac2f188e97f50eb7
Member ac2f188e97f50eb7 removed from cluster 194cd14a48430083

再启动etcd服务

# systemctl start etcd

报错：

etcd: error validating peerURLs {ClusterID:194cd14a48430083 Members:[&{ID:1ce6d6d01109192 RaftAttributes:{PeerURLs:[https://192.168.0.102:2380]} Attributes:{Name:etcd03 ClientURLs:[https://192.168.0.102:2379]}} &{ID:9b534175b46ea789 RaftAttributes:{PeerURLs:[https://192.168.0.100:2380]} Attributes:{Name:etcd01 ClientURLs:[https://192.168.0.100:2379]}}] RemovedMemberIDs:[]}: member count is unequal

报错：

etcd: the member has been permanently removed from the cluster the data-dir used by this member must be removed

4. etcd恢复数据

在etcd02节点恢复一下数据试试：

# mv /var/lib/etcd/member /var/lib/member
# rm -rf /var/lib/etcd/*
# etcdctl snapshot restore /var/lib/member/snap/db --skip-hash-check=true
2018-06-22 11:28:35.622666 I | mvcc: restore compact to 10177401
2018-06-22 11:28:35.659626 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32
# systemctl start etcd

服务启动了，自己把自己选做主，服务倒是启动了，加入集群还是出错，用正常的节点备份再恢复

# etcdctl snapshot save etcdback.db

# etcdctl member add etcd02 http://192.168.0.101:2380
Error: member name not provided.

看看现在集群的其他2个etcd

# curl -k --key /etc/kubernetes/ssl/etcd-key.pem --cert /etc/kubernetes/ssl/etcd.pem https://192.168.0.100:2380/members
[{"id":130161754177048978,"peerURLs":["https://192.168.0.102:2380"],"name":"etcd03","clientURLs":["https://192.168.0.102:2379"]},{"id":11192361472739944329,"peerURLs":["https://192.168.0.100:2380"],"name":"etcd01","clientURLs":["https://192.168.0.100:2379"]}]

参考文档：

etcdctl member add etcd_name –peer-urls=”https://peerURLs”

再次添加

# etcdctl member add etcd02 --peer-urls="https://192.168.0.101:2380"
Member 41c2a7b938a5e387 added to cluster 194cd14a48430083

ETCD_NAME="etcd02"
ETCD_INITIAL_CLUSTER="etcd03=https://192.168.0.102:2380,etcd02=https://192.168.0.101:2380,etcd01=https://192.168.0.100:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

查看etcd member状态：

# etcdctl member list
1ce6d6d01109192, started, etcd03, https://192.168.0.102:2380, https://192.168.0.102:2379
9b534175b46ea789, started, etcd01, https://192.168.0.100:2380, https://192.168.0.100:2379
ad17c3da831c84c7, unstarted, , https://192.168.0.101:2380,

报错：

etcd: request cluster ID mismatch (got 194cd14a48430083 want cdf818194e3a8c32)

发现步骤顺序错误，应该是先添加到etcd集群，再启动etcd服务，我们现在先启动etcd服务，就是一个etcd单点

etcd节点加入集群

故障的etcd主机

# systemctl stop etcd

正常的etcd主机：

# etcdctl member remove ad17c3da831c84c7

# etcdctl member add etcd02 --peer-urls="https://192.168.0.101:2380"
Member 41c2a7b938a5e387 added to cluster 194cd14a48430083

ETCD_NAME="etcd02"
ETCD_INITIAL_CLUSTER="etcd03=https://192.168.0.102:2380,etcd02=https://192.168.0.101:2380,etcd01=https://192.168.0.100:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

故障的etcd主机，查看现在etcd状态：

# etcdctl endpoint health
127.0.0.1:2379 is healthy: successfully committed proposal: took = 24.52485ms

# etcdctl member list
1ce6d6d01109192, started, etcd03, https://192.168.0.102:2380, https://192.168.0.102:2379
41c2a7b938a5e387, started, etcd02, https://192.168.0.101:2380, https://192.168.0.101:2379
9b534175b46ea789, started, etcd01, https://192.168.0.100:2380, https://192.168.0.100:2379

到这里，etcd故障修复完毕

5. etcd常用命令

查看状态

# export ETCDCTL_API=3

# etcdctl endpoint status --write-out=table
+----------------+------------------+---------+---------+-----------+-----------+------------+
|    ENDPOINT    |        ID        | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+----------------+------------------+---------+---------+-----------+-----------+------------+
| 127.0.0.1:2379 | 41c2a7b938a5e387 |  3.2.15 |   15 MB |      true |       317 |   11051403 |
+----------------+------------------+---------+---------+-----------+-----------+------------+

备份及恢复

etcdctl snapshot save etcdback.db
etcdctl snapshot status etcdback.db --write-out=table
etcdctl snapshot restore etcdback.db --skip-hash-check=true

etcd监控

# curl -L http://localhost:2379/metrics
# HELP etcd_debugging_mvcc_keys_total Total number of keys.
# TYPE etcd_debugging_mvcc_keys_total gauge
etcd_debugging_mvcc_keys_total 776
# HELP etcd_debugging_mvcc_pending_events_total Total number of pending events to be sent.
# TYPE etcd_debugging_mvcc_pending_events_total gauge
etcd_debugging_mvcc_pending_events_total 0
# HELP etcd_debugging_mvcc_put_total Total number of puts seen by this member.
# TYPE etcd_debugging_mvcc_put_total counter
etcd_debugging_mvcc_put_total 9.548201e+06
# HELP etcd_debugging_mvcc_range_total Total number of ranges seen by this member.
# TYPE etcd_debugging_mvcc_range_total counter
etcd_debugging_mvcc_range_total 2.1052143e+07
# HELP etcd_debugging_mvcc_slow_watcher_total Total number of unsynced slow watchers.
# TYPE etcd_debugging_mvcc_slow_watcher_total gauge
etcd_debugging_mvcc_slow_watcher_total 0
# HELP etcd_debugging_mvcc_txn_total Total number of txns seen by this member.
# TYPE etcd_debugging_mvcc_txn_total counter
etcd_debugging_mvcc_txn_total 0
# HELP etcd_debugging_mvcc_watch_stream_total Total number of watch streams.
# TYPE etcd_debugging_mvcc_watch_stream_total gauge
etcd_debugging_mvcc_watch_stream_total 125
# HELP etcd_debugging_mvcc_watcher_total Total number of watchers.
# TYPE etcd_debugging_mvcc_watcher_total gauge
etcd_debugging_mvcc_watcher_total 125
# HELP etcd_debugging_server_lease_expired_total The total number of expired leases.
# TYPE etcd_debugging_server_lease_expired_total counter
etcd_debugging_server_lease_expired_total 3649

适合用prometheus监控

global:
  scrape_interval: 10s
scrape_configs:
  - job_name: etcd
    static_configs:
    - targets: ['192.168.0.100:2379','192.168.0.101:2379','192.168.0.102:2379']

图解raft算法 http://thesecretlivesofdata.com/raft/

etcd获取kubernetes的数据

# export ETCDCTL_API=3
# etcdctl get /registry/namespaces/default --prefix -w json|python -m json.tool
{
    "count": 1,
    "header": {
        "cluster_id": 1823062066148343939,
        "member_id": 11192361472739944329,
        "raft_term": 317,
        "revision": 10880816
    },
    "kvs": [
        {
            "create_revision": 6,
            "key": "L3JlZ2lzdHJ5L25hbWVzcGFjZXMvZGVmYXVsdA==",
            "mod_revision": 6,
            "value": "azhzAAoPCgJ2MRIJTmFtZXNwYWNlEl8KRQoHZGVmYXVsdBIAGgAiACokOTVlNzdjMWEtM2Q1Ny0xMWU4LTk5YzItMDA1MDU2YmU3NWEzMgA4AEIICK7qttYFEAB6ABIMCgprdWJlcm5ldGVzGggKBkFjdGl2ZRoAIgA=",
            "version": 1
        }
    ]
}

查看key的内容
# echo L3JlZ2lzdHJ5L25hbWVzcGFjZXMvZGVmYXVsdA== |base64 -d
/registry/namespaces/default

#!/bin/bash
# Get kubernetes keys from etcd
export ETCDCTL_API=3
keys=`etcdctl get /registry --prefix -w json|python -m json.tool|grep key|cut -d ":" -f2|tr -d '"'|tr -d ","`
for x in $keys;do
  echo $x|base64 -d|sort
done

获取etcd中kubernetes所有对象的key

ETCD数据库异常：mvcc: database space exceeded解决

• 问题来源：在k8s集群中给node打标签发现报错

[root@master1]# kubectl label node  30.4.228.20 env=prod
Error from server: etcdserver: mvcc: database space exceeded

• 环境信息

etcd集群：30.4.228.19，30.4.228.20，30.4.228.22 （配置了安全加密）

原因分析

• etcd服务未设置自动压缩参数（auto-compact）
• etcd 默认不会自动 compact，需要设置启动参数，或者通过命令进行compact，如果变更频繁建议设置，否则会导致空间和内存的浪费以及错误。Etcd v3 的默认的 backend quota 2GB，如果不 compact，boltdb 文件大小超过这个限制后，就会报错：”Error: etcdserver: mvcc: database space exceeded”，导致数据无法写入。

处理过程

1、 获取旧版本号:

[root@etcd1]# rev=$(/usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem 
--cert=/etc/etcd/ssl/etcd.pem  
--key=/etc/etcd/ssl/etcd-key.pem  
--endpoints="https://127.0.0.1:2379" 
endpoint status --write-out="json"  
| egrep -o '"revision":[0-9]*'  
| egrep -o '[0-9].*') 

[root@etcd1]# echo $rev 

2 、压缩旧版本

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.20:2379" compact $rev

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.20:2379" defrag 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.20:2379" alarm list 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.19:2379" compact $rev

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.19:2379" defrag 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.22:2379" compact $rev

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.22:2379" defrag 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.22:2379" alarm list 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.22:2379" alarm disarm

3、查看告警

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.22:2379" alarm list

[root@etcd1]#  /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.19:2379" alarm list 

[root@etcd1]# /usr/local/bin/etcdctl --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://30.4.228.20:2379" alarm list

etcd相关命令

1、设置etcd配额：

# 设置16MB的配额
etcd --quota-backend-bytes=$((16*1024*1024))

2、触发配额耗尽：

while [1];do dd if=dev/urandom bs=1024 count=1024 
 | ETCDCTL_API=3 etcdctl put key || break; done
...
Error: rpc error: code = 8 desc = etcdserver: mvcc:database space exceeded

3、确认数据空间超出配额：

ETCDCTL_API=3 etcdctl --write-out=table endpoint status
----------------+------------------+-----------+---------+-----------+-----------+------------+
|    ENDPOINT    |        ID        |  VERSION  | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
+----------------+------------------+-----------+---------+-----------+-----------+------------+
| 127.0.0.1:2379 | bf9071f4639c75cc | 2.3.0+git | 18 MB   | true      |         2 |       3332 |
+----------------+------------------+-----------+---------+-----------+-----------+------------+

4、查看告警：

ETCDCTL_API=3 etcdctl alarm list

5、整合压缩、碎片整理：

1) 获取当前etcd数据的修订版本(revision)

rev=$(ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*')

2) 整合压缩旧版本数据

ETCDCTL_API=3 etcdctl compact $rev

3) 执行碎片整理

ETCDCTL_API=3 etcdctl defrag

4) 解除告警

ETCDCTL_API=3 etcdctl alarm disarm

5) 备份以及查看备份数据信息

ETCDCTL_API=3 etcdctl snapshot save backup.db
ETCDCTL_API=3 etcdctl snapshot status backup.db

主要两步：

• 添加节点
• 启动新节点

原本etcd集群

# etcdctl member list
b7124c8d88451: name=myetcd1 peerURLs=http://192.168.9.100:2380 clientURLs=http://192.168.9.100:2379 isLeader=true
235dcf74ed6248d5: name=myetcd3 peerURLs=http://192.168.9.100:2382 clientURLs=http://192.168.9.100:2399 isLeader=false
e35665335259ca10: name=myetcd2 peerURLs=http://192.168.9.100:2381 clientURLs=http://192.168.9.100:2389 isLeader=false

新增节点： 192.168.9.101：2383

# etcdctl member add myetcd4  http://192.168.9.101:2383

Added member named myetcd4 with ID 2644935c1a10c721 to cluster

ETCD_NAME="myetcd4"
ETCD_INITIAL_CLUSTER="myetcd1=http://192.168.9.100:2380,myetcd3=http://192.168.9.100:2382,myetcd4=http://192.168.9.101:2383,myetcd2=http://192.168.9.100:2381"
ETCD_INITIAL_CLUSTER_STATE="existing"

启动新节点：

# etcd --name myetcd4  --listen-client-urls http://0.0.0.0:2409 --advertise-client-urls http://192.168.9.101:2409 --listen-peer-urls http://0.0.0.0:2383 --initial-advertise-peer-urls http://192.168.9.101:2383  --initial-cluster-token etcd-cluster-test --initial-cluster-state existing --initial-cluster myetcd1=http://192.168.9.100:2380,myetcd2=http://192.168.9.100:2381,myetcd3=http://192.168.9.100:2382,myetcd4=http://192.168.9.101:2383
1

下载安装

• 从这下载https://github.com/coreos/etcd/releases/download/v3.3.2/etcd-v3.3.2-linux-amd64.tar.gz
• tar xzvf etcd-v3.3.2-linux-amd64.tar.gz
• cd etcd-v3.3.2-linux-amd64; cp etcd* /user/local/bin/
• 这样即成功添加etcd命令
  etcd –version

运行与搭建

常见命令演示

etcd –version
etcdctl –version

API3的要这样
ETCDCTL_API=3 etcdctl version

启动：etcd
写一个数据与读取一个数据：key:key1 value:helloworld
ETCDCTL_API=3 etcdctl –endpoints=localhost:2379 put helloworld
ETCDCTL_API=3 etcdctl –endpoints=localhost:2379 get key1

单机启动：

etcd --name myetcd1  --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://0.0.0.0:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://0.0.0.0:2380  --initial-cluster my-etcd-1=http://0.0.0.0:2380

集群启动：

etcd --name myetcd1  --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://192.168.9.100:2379 --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://192.168.9.100:2380  --initial-cluster-token etcd-cluster-test --initial-cluster-state new --initial-cluster myetcd1=http://192.168.9.100:2380,myetcd2=http://192.168.9.100:2381,myetcd3=http://192.168.9.100:2382

etcd --name myetcd2  --listen-client-urls http://0.0.0.0:2389 --advertise-client-urls http://192.168.9.100:2389 --listen-peer-urls http://0.0.0.0:2381 --initial-advertise-peer-urls http://192.168.9.100:2381  --initial-cluster-token etcd-cluster-test --initial-cluster-state new --initial-cluster myetcd1=http://192.168.9.100:2380,myetcd2=http://192.168.9.100:2381,myetcd3=http://192.168.9.100:2382

etcd --name myetcd3  --listen-client-urls http://0.0.0.0:2399 --advertise-client-urls http://192.168.9.100:2399 --listen-peer-urls http://0.0.0.0:2382 --initial-advertise-peer-urls http://192.168.9.100:2382  --initial-cluster-token etcd-cluster-test --initial-cluster-state new --initial-cluster myetcd1=http://192.168.9.100:2380,myetcd2=http://192.168.9.100:2381,myetcd3=http://192.168.9.100:2382

用三个端口2380,2381,2382来模拟集群(这三个是成员之间通信),2379,2389,2399是给客户端连接的.服务器IP：192.168.9.100, 如果在本机模拟集群, 可以将192.168.9.100改为0.0.0.0

带advertise参数是广播参数: 如–listen-client-urls和–advertise-client-urls, 前者是Etcd端监听客户端的url,后者是Etcd客户端请求的url, 两者端口是相同的, 只不过后者一般为公网IP, 暴露给外部使用.

查看成员:

etcdctl member list 

使用时需要指定endpoints(默认本地端口2379), 集群时数据会迅速同步:

ETCDCTL_API=3 etcdctl –endpoints=127.0.0.1:2389 put key1 xx 
ETCDCTL_API=3 etcdctl –endpoints=127.0.0.1:2379 get key1

参数说明

–name etcd0：本member的名字

–initial-advertise-peer-urls http://192.168.9.100:2380： 其他member使用，其他member通过该地址与本member交互信息。一定要保证从其他member能可访问该地址。静态配置方式下，该参数的value一定要同时在–initial-cluster参数中存在。memberID的生成受–initial-cluster-token和–initial-advertise-peer-urls影响。

–listen-peer-urls http://0.0.0.0:2380：本member侧使用，用于监听其他member发送信息的地址。ip为全0代表监听本member侧所有接口

–listen-client-urls http://0.0.0.0:2379： 本member侧使用，用于监听etcd客户发送信息的地址。ip为全0代表监听本member侧所有接口

–advertise-client-urls http://192.168.9.100:2379： etcd客户使用，客户通过该地址与本member交互信息。一定要保证从客户侧能可访问该地址

–initial-cluster-token etcd-cluster-2：用于区分不同集群。本地如有多个集群要设为不同。

–initial-cluster myetcd0=http://192.168.9.100:2380,myetcd1=http://192.168.9.100:2381,myetcd2=http://192.168.9.100:2382：本member侧使用。描述集群中所有节点的信息，本member根据此信息去联系其他member。memberID的生成受–initial-cluster-token和–initial-advertise-peer-urls影响。

–initial-cluster-state new：用于指示本次是否为新建集群。有两个取值new和existing。如果填为existing，则该member启动时会尝试与其他member交互。集群初次建立时，要填为new，经尝试最后一个节点填existing也正常，其他节点不能填为existing。集群运行过程中，一个member故障后恢复时填为existing，经尝试填为new也正常。

–data-dir：指定节点的数据存储目录，这些数据包括节点ID，集群ID，集群初始化配置，Snapshot文件，若未指定-wal-dir，还会存储WAL文件；如果不指定会用缺省目录。

–discovery http://192.168.9.100:20003/v2/keys/discovery/78b12ad7-2c1d-40db-9416-3727baf686cb：用于自发现模式下，指定第三方etcd上key地址，要建立的集群各member都会向其注册自己的地址。

使用详细说明

ETCD API有两种, 一种是3, 一种是2, 默认为2, 我们主要用3:

API3:

[root@guods-1 centos]# ETCDCTL_API=3 etcdctl --endpoints=localhost:2379 put key1 helloworld
OK
[root@guods-1 centos]# ETCDCTL_API=3 etcdctl --endpoints=localhost:2379 get key1
key1
helloworld

API2:

[root@guods-1 centos]# etcdctl set key2 helloworld
helloworld
[root@guods-1 centos]# etcdctl get key2
helloworld

命令详解：

[root@guods-1 centos]# ETCDCTL_API=2 etcdctl
NAME:
   etcdctl - A simple command line client for etcd.

USAGE:
   etcdctl [global options] command [command options] [arguments...]

VERSION:
   3.3.2

COMMANDS:
     backup          backup an etcd directory
     cluster-health  check the health of the etcd cluster
     mk              make a new key with a given value
     mkdir           make a new directory
     rm              remove a key or a directory
     rmdir           removes the key if it is an empty directory or a key-value pair
     get             retrieve the value of a key
     ls              retrieve a directory
     set             set the value of a key
     setdir          create a new directory or update an existing directory TTL
     update          update an existing key with a given value
     updatedir       update an existing directory
     watch           watch a key for changes
     exec-watch      watch a key for changes and exec an executable
     member          member add, remove and list subcommands
     user            user add, grant and revoke subcommands
     role            role add, grant and revoke subcommands
     auth            overall auth controls
     help, h         Shows a list of commands or help for one command

GLOBAL OPTIONS:
   --debug                          output cURL commands which can be used to reproduce the request
   --no-sync                        don't synchronize cluster information before sending request
   --output simple, -o simple       output response in the given format (simple, `extended` or `json`) (default: "simple")
   --discovery-srv value, -D value  domain name to query for SRV records describing cluster endpoints
   --insecure-discovery             accept insecure SRV records describing cluster endpoints
   --peers value, -C value          DEPRECATED - "--endpoints" should be used instead
   --endpoint value                 DEPRECATED - "--endpoints" should be used instead
   --endpoints value                a comma-delimited list of machine addresses in the cluster (default: "http://127.0.0.1:2379,http://127.0.0.1:4001")
   --cert-file value                identify HTTPS client using this SSL certificate file
   --key-file value                 identify HTTPS client using this SSL key file
   --ca-file value                  verify certificates of HTTPS-enabled servers using this CA bundle
   --username value, -u value       provide username[:password] and prompt if password is not supplied.
   --timeout value                  connection timeout per request (default: 2s)
   --total-timeout value            timeout for the command execution (except watch) (default: 5s)
   --help, -h                       show help
   --version, -v                    print the version

[root@guods-1 centos]# ETCDCTL_API=3 etcdctl
NAME:
    etcdctl - A simple command line client for etcd3.

USAGE:
    etcdctl

VERSION:
    3.3.2

API VERSION:
    3.3


COMMANDS:
    get         Gets the key or a range of keys
    put         Puts the given key into the store
    del         Removes the specified key or range of keys [key, range_end)
    txn         Txn processes all the requests in one transaction
    compaction      Compacts the event history in etcd
    alarm disarm        Disarms all alarms
    alarm list      Lists all alarms
    defrag          Defragments the storage of the etcd members with given endpoints
    endpoint health     Checks the healthiness of endpoints specified in `--endpoints` flag
    endpoint status     Prints out the status of endpoints specified in `--endpoints` flag
    endpoint hashkv     Prints the KV history hash for each endpoint in --endpoints
    move-leader     Transfers leadership to another etcd cluster member.
    watch           Watches events stream on keys or prefixes
    version         Prints the version of etcdctl
    lease grant     Creates leases
    lease revoke        Revokes leases
    lease timetolive    Get lease information
    lease list      List all active leases
    lease keep-alive    Keeps leases alive (renew)
    member add      Adds a member into the cluster
    member remove       Removes a member from the cluster
    member update       Updates a member in the cluster
    member list     Lists all members in the cluster
    snapshot save       Stores an etcd node backend snapshot to a given file
    snapshot restore    Restores an etcd member snapshot to an etcd directory
    snapshot status     Gets backend snapshot status of a given file
    make-mirror     Makes a mirror at the destination etcd cluster
    migrate         Migrates keys in a v2 store to a mvcc store
    lock            Acquires a named lock
    elect           Observes and participates in leader election
    auth enable     Enables authentication
    auth disable        Disables authentication
    user add        Adds a new user
    user delete     Deletes a user
    user get        Gets detailed information of a user
    user list       Lists all users
    user passwd     Changes password of user
    user grant-role     Grants a role to a user
    user revoke-role    Revokes a role from a user
    role add        Adds a new role
    role delete     Deletes a role
    role get        Gets detailed information of a role
    role list       Lists all roles
    role grant-permission   Grants a key to a role
    role revoke-permission  Revokes a key from a role
    check perf      Check the performance of the etcd cluster
    help            Help about any command

OPTIONS:
      --cacert=""               verify certificates of TLS-enabled secure servers using this CA bundle
      --cert=""                 identify secure client using this TLS certificate file
      --command-timeout=5s          timeout for short running command (excluding dial timeout)
      --debug[=false]               enable client-side debug logging
      --dial-timeout=2s             dial timeout for client connections
  -d, --discovery-srv=""            domain name to query for SRV records describing cluster endpoints
      --endpoints=[127.0.0.1:2379]      gRPC endpoints
  -h, --help[=false]                help for etcdctl
      --hex[=false]             print byte strings as hex encoded strings
      --insecure-discovery[=true]       accept insecure SRV records describing cluster endpoints
      --insecure-skip-tls-verify[=false]    skip server certificate verification
      --insecure-transport[=true]       disable transport security for client connections
      --keepalive-time=2s           keepalive time for client connections
      --keepalive-timeout=6s            keepalive timeout for client connections
      --key=""                  identify secure client using this TLS key file
      --user=""                 username[:password] for authentication (prompt if password is not supplied)
  -w, --write-out="simple"          set the output format (fields, json, protobuf, simple, table)

一、简介

“A highly-available key value store for shared configuration and service discovery.”

Etcd是coreos开发的分布式服务系统，内部采用raft协议作为一致性算法。作为一个高可用的配置共享、服务发现的键值存储系统，Etcd有以下的特点：

• 简单：安装配置简单，而且提供了 HTTP API 进行交互，使用也很简单
• 安全：支持 SSL 证书验证
• 快速：根据官方提供的数据，单实例支持每秒2k+读操作、1k写操作
• 可靠：采用raft算法，实现分布式系统数据的可用性和一致性

Etcd构建自身高可用集群主要有三种形式:

• 静态发现: 预先已知 Etcd 集群中有哪些节点，在启动时直接指定好Etcd的各个node节点地址
• Etcd动态发现: 通过已有的Etcd集群作为数据交互点，然后在扩展新的集群时实现通过已有集群进行服务发现的机制
• DNS动态发现: 通过DNS查询方式获取其他节点地址信息

本文主要介绍第一种方式，后续会陆续介绍剩下的两种方式。(直接docker安装请移步：quay.io/coreos/etcd 基于Docker镜像的集群搭建)

二、环境介绍

三台虚拟机，系统环境均为Centos7，对应节点名称及IP地址如下：

• node1:192.168.7.163
• node2:192.168.7.57
• etcd2:192.168.7.58

首先将这个信息添加到三台主机的hosts文件中，编辑/etc/hosts，填入以下信息：

• 192.168.7.163 node1
• 192.168.7.57 node2
• 192.168.7.58 etcd2

三、安装、配置Etcd

# yum install etcd -y

yum安装的etcd默认配置文件在/etc/etcd/etcd.conf，以下将三个节点上的配置贴出来，请注意不同点（未贴出的，则表明不需要更改）

node1

# [member]
# 节点名称
ETCD_NAME=node1
# 数据存放位置
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
#ETCD_SNAPSHOT_COUNT="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
# 监听其他 Etcd 实例的地址
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
# 监听客户端地址
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379,http://0.0.0.0:4001"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#
#[cluster]
# 通知其他 Etcd 实例地址
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://node1:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
# 初始化集群内节点地址
ETCD_INITIAL_CLUSTER="node1=http://node1:2380,node2=http://node2:2380,etcd2=http://etcd2:2380"
# 初始化集群状态，new 表示新建
ETCD_INITIAL_CLUSTER_STATE="new"
# 初始化集群 token
ETCD_INITIAL_CLUSTER_TOKEN="mritd-etcd-cluster"
# 通知 客户端地址
ETCD_ADVERTISE_CLIENT_URLS="http://node1:2379,http://node1:4001"

node2

# [member]
ETCD_NAME=node2
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
#ETCD_SNAPSHOT_COUNT="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379,http://0.0.0.0:4001"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#
#[cluster]

ETCD_INITIAL_ADVERTISE_PEER_URLS="http://node2:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="node1=http://node1:2380,node2=http://node2:2380,etcd2=http://etcd2:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="mritd-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="http://node2:2379,http://node2:4001"

etcd2

# [member]
ETCD_NAME=etcd2
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
#ETCD_SNAPSHOT_COUNT="10000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379,http://0.0.0.0:4001"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
#ETCD_CORS=""
#
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://etcd2:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="node1=http://node1:2380,node2=http://node2:2380,etcd2=http://etcd2:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="mritd-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="http://etcd2:2379,http://etcd2:4001"

改好配置之后，在各个节点上开启etcd服务：

# systemctl restart etcd

四、测试验证

[root@localhost ~]# etcdctl set testdir/testkey0 0
0
[root@localhost ~]# etcdctl set testdir/testkey1 1
1
[root@localhost ~]# etcdctl set testdir/testkey2 2
2
[root@localhost ~]# etcdctl ls
/test
/testdir
[root@localhost ~]# etcdctl ls testdir
/testdir/testkey0
/testdir/testkey1
/testdir/testkey2
[root@localhost ~]# etcdctl get testdir/testkey2
2
[root@localhost ~]# etcdctl member list
377aa10974e8238d: name=node1 peerURLs=http://node1:2380 clientURLs=http://node1:2379,http://node1:4001 isLeader=true
9de2d4fdbbd835b6: name=etcd2 peerURLs=http://etcd2:2380 clientURLs=http://etcd2:2379,http://etcd2:4001 isLeader=false
f75ed833c7cbbe65: name=node2 peerURLs=http://node2:2380 clientURLs=http://node2:2379,http://node2:4001 isLeader=false