---
# System prepended metadata

title: '[2020.05.18] ETCD cluster'
tags: [ETCD]

---

# [2020.05.18] ETCD cluster
[toc]
>[name=weng yichiun]
###### tags: `ETCD`
- 3個node做cluster
- 開啟tls功能
- 三台vm各自用docker
- 192.168.56.131-133
```= 
/etc/hosts
127.0.0.1 localhost
127.0.1.1 k8s_etcd03
192.168.56.131 k8s_etcd01
192.168.56.132 k8s_etcd02
192.168.56.133 k8s_etcd03
```
## 憑證
### CA
```=
mkdir -p /etc/etcd-certs/
cd /etc/etcd-certs/

cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json

vim ca-config.json
```
- ca-config
:::spoiler ca-config.json
```json=
{
    "signing": {
        "default": {
            "expiry": "438000h"
        },
        "profiles": {
            "server": {
                "expiry": "438000h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "438000h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {
                "expiry": "438000h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth",
                    "server auth"
                ]
            }
        }
    }
}

```
:::
- ca-csr
```=
vim ca-csr.json
```
:::spoiler ca-csr.json
```json=
{
    "CN": "etcd",
    "key": {
        "algo": "ecdsa",
        "size": 256
    }
}

```
:::

- 產生CA
```=
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
```
- 產生三個檔案
    - ca.csr, 證書請求文件，實際上是cfssl的一個中間文件，由上述配置文件生成，其中的關鍵信息是通信實體的基本信息和一對公私密鑰，這些內容將用於簽發證書。
    - ca.pem, 證書文件，即上面提到的根CA證書，需要保存到所有服務器和客戶端上，用於驗證通信過程中交互的證書。與CA證書是自簽證書，其實就是用證書請求文件中的私鑰對其他信息包括公鑰簽名的過程。
    - ca-key.pem, 私鑰文件，可用於驗證ca.pem, 日常通信不會使用，應該離線保存。
```
root@k8s_etcd03:/etc/kubernetes/etcd2# ll
total 28
drwxr-xr-x 2 root root 4096 May 12 12:18 ./
drwxr-xr-x 4 root root 4096 May 11 12:03 ../
-rw-r--r-- 1 root root  582 May 12 12:16 ca-config.json
-rw-r--r-- 1 root root  387 May 12 12:18 ca.csr
-rw-r--r-- 1 root root  176 May 12 12:17 ca-csr.json
-rw------- 1 root root  227 May 12 12:18 ca-key.pem
-rw-r--r-- 1 root root  664 May 12 12:18 ca.pem
```

### 建立server cert
- gen server cert
```=
cfssl print-defaults csr > server.json
vim server.json
```
:::spoiler server.json
```json=
{
    "CN": "etcd",
    "hosts": [
        "192.168.56.131",
        "192.168.56.132",
        "192.168.56.133",
        "127.0.0.1",
        "localhost",
        "etcd"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "US",
            "L": "CA",
            "ST": "San Francisco"
        }
    ]
}

```
:::

- gen cert , private key
```
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
```

```
root@k8s_etcd03:/etc/etcd-certs# ll
total 44
drwxr-xr-x 2 root root 4096 May 12 15:04 ./
drwxr-xr-x 4 root root 4096 May 11 12:03 ../
-rw-r--r-- 1 root root  577 May 12 15:01 ca-config.json
-rw-r--r-- 1 root root  347 May 12 15:01 ca.csr
-rw-r--r-- 1 root root   87 May 12 15:01 ca-csr.json
-rw------- 1 root root  227 May 12 15:01 ca-key.pem
-rw-r--r-- 1 root root  591 May 12 15:01 ca.pem
-rw-r--r-- 1 root root  481 May 12 15:04 server.csr
-rw-r--r-- 1 root root  278 May 12 15:04 server.json
-rw------- 1 root root  227 May 12 15:04 server-key.pem
-rw-r--r-- 1 root root  721 May 12 15:04 server.pem
```
### 建立client cert
```
cfssl print-defaults csr > client.json
vim client.json
```
:::spoiler client.json
```
{
    "CN": "etcd-client",
    "key": {
        "algo": "ecdsa",
        "size": 256
    }
}

```
:::
```
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
```
### 建立peer之間的cert
```
cfssl print-defaults csr > etcd01-peer-csr.json
vim etcd01-peer-csr.json
```
:::spoiler etcd01-peer-csr.json
```
{
    "CN": "etcd01-peer",
    "hosts": [
        "192.168.56.131",
        "192.168.56.132",
        "192.168.56.133",
        "127.0.0.1",
        "localhost",
        "etcd"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "US",
            "L": "CA",
            "ST": "San Francisco"
        }
    ]
}
```
:::
- gen
```=
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd01-peer-csr.json | cfssljson -bare etcd01-peer


-rw-r--r--  1 root root  489 May 15 16:18 etcd01-peer.csr
-rw-r--r--  1 root root  315 May 15 16:13 etcd01-peer-csr.json
-rw-------  1 root root  227 May 15 16:18 etcd01-peer-key.pem
-rw-r--r--  1 root root  745 May 15 16:18 etcd01-peer.pem
```

## cluster node
- 3台vm各自安裝docker
```=
sudo apt install docker.io -y
docker pull quay.io/coreos/etcd:v3.3.13
```
- 跟node1 拿server憑證, ca,peer憑證
```
scp webuser@node_01:/etc/etcd-certs/server*.pem /etc/etcd-certs/
scp webuser@node_01:/etc/etcd-certs/ca*.pem /etc/etcd-certs/
scp webuser@node_01:/etc/etcd-certs/etcd01*.pem /etc/etcd-certs/
```
### docker-compose yaml
#### node1 192.168.56.131
```=
version: '3'

services:
  etcd:
    image: quay.io/coreos/etcd:v3.3.13
    network_mode: "host"
    volumes:
      - /etc/etcd-certs/:/etc/etcd-certs/
      - /root/etcd_cluster/:/etcd-data/
    environment:
      - ETCD_NAME=etcd_01
      - ETCD_DATA_DIR=/etcd-data/
      - ETCD_INITIAL_CLUSTER=etcd_01=https://192.168.56.131:2380,etcd_02=https://192.168.56.132:2380,etcd_03=https://192.168.56.133:2380
      - ETCD_INITIAL_CLUSTER_STATE=new
      - ETCD_INITIAL_CLUSTER_TOKEN=ycw-etcd-token
      - ETCD_LISTEN_CLIENT_URLS=https://192.168.56.131:2379,https://127.0.0.1:2379
      - ETCD_ADVERTISE_CLIENT_URLS=https://192.168.56.131:2379
      - ETCD_LISTEN_PEER_URLS=https://192.168.56.131:2380
      - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.56.131:2380
      - ETCD_CERT_FILE=/etc/etcd-certs/server.pem
      - ETCD_KEY_FILE=/etc/etcd-certs/server-key.pem
      - ETCD_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_CLIENT_CERT_AUTH
      - ETCD_CLIENT_CRL_FILE=/etc/etcd-certs/client.pem
      - ETCD_PEER_CERT_FILE=/etc/etcd-certs/etcd01-peer.pem
      - ETCD_PEER_KEY_FILE=/etc/etcd-certs/etcd01-peer-key.pem
      - ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_PEER_CLIENT_CERT_AUTH
    ports:
      - "2379:2379"
      - "2380:2380"
    container_name: etcd

```
#### node2 192.168.56.132
```=
version: '3'

services:
  etcd:
    image: quay.io/coreos/etcd:v3.3.13
    network_mode: "host"
    volumes:
      - /etc/etcd-certs/:/etc/etcd-certs/
      - /root/etcd_cluster/:/etcd-data/
    environment:
      - ETCD_NAME=etcd_02
      - ETCD_DATA_DIR=/etcd-data/
      - ETCD_INITIAL_CLUSTER=etcd_01=https://192.168.56.131:2380,etcd_02=https://192.168.56.132:2380,etcd_03=https://192.168.56.133:2380
      - ETCD_INITIAL_CLUSTER_STATE=new
      - ETCD_INITIAL_CLUSTER_TOKEN=ycw-etcd-token
      - ETCD_LISTEN_CLIENT_URLS=https://192.168.56.132:2379,https://127.0.0.1:2379
      - ETCD_ADVERTISE_CLIENT_URLS=https://192.168.56.132:2379
      - ETCD_LISTEN_PEER_URLS=https://192.168.56.132:2380
      - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.56.132:2380
      - ETCD_CERT_FILE=/etc/etcd-certs/server.pem
      - ETCD_KEY_FILE=/etc/etcd-certs/server-key.pem
      - ETCD_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_CLIENT_CERT_AUTH
      - ETCD_CLIENT_CRL_FILE=/etc/etcd-certs/client.pem
      - ETCD_PEER_CERT_FILE=/etc/etcd-certs/etcd01-peer.pem
      - ETCD_PEER_KEY_FILE=/etc/etcd-certs/etcd01-peer-key.pem
      - ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_PEER_CLIENT_CERT_AUTH
    ports:
      - "2379:2379"
      - "2380:2380"
    container_name: etcd

```
#### node3 192.168.56.133
```=
version: '3'

services:
  etcd:
    image: quay.io/coreos/etcd:v3.3.13
    network_mode: "host"
    volumes:
      - /etc/etcd-certs/:/etc/etcd-certs/
      - /root/etcd_cluster/:/etcd-data/
    environment:
      - ETCD_NAME=etcd_03
      - ETCD_DATA_DIR=/etcd-data/
      - ETCD_INITIAL_CLUSTER=etcd_01=https://192.168.56.131:2380,etcd_02=https://192.168.56.132:2380,etcd_03=https://192.168.56.133:2380
      - ETCD_INITIAL_CLUSTER_STATE=new
      - ETCD_INITIAL_CLUSTER_TOKEN=ycw-etcd-token
      - ETCD_LISTEN_CLIENT_URLS=https://192.168.56.133:2379,https://127.0.0.1:2379
      - ETCD_ADVERTISE_CLIENT_URLS=https://192.168.56.133:2379
      - ETCD_LISTEN_PEER_URLS=https://192.168.56.133:2380
      - ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.56.133:2380
      - ETCD_CERT_FILE=/etc/etcd-certs/server.pem
      - ETCD_KEY_FILE=/etc/etcd-certs/server-key.pem
      - ETCD_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_CLIENT_CERT_AUTH
      - ETCD_CLIENT_CRL_FILE=/etc/etcd-certs/client.pem
      - ETCD_PEER_CERT_FILE=/etc/etcd-certs/etcd01-peer.pem
      - ETCD_PEER_KEY_FILE=/etc/etcd-certs/etcd01-peer-key.pem
      - ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd-certs/ca.pem
      - ETCD_PEER_CLIENT_CERT_AUTH
    ports:
      - "2379:2379"
      - "2380:2380"
    container_name: etcd


```
### 運行
- 三台啟動
`docker-compose up -d`
- 對任何一台put/set(看版本) 3.4以後是put
```=
docker  exec -it etcd etcdctl --cert-file /etc/etcd-certs/client.pem --key-file /etc/etcd-certs/client-key.pem  -ca-file /etc/etcd-certs/ca.pem --endpoints https://127.0.0.1:2379 set foo bar
```
- get 
```=
docker  exec -it etcd etcdctl --cert-file /etc/etcd-certs/client.pem --key-file /etc/etcd-certs/client-key.pem  -ca-file /etc/etcd-certs/ca.pem --endpoints https://127.0.0.1:2379 get foo
docker  exec -it etcd etcdctl --cert-file /etc/etcd-certs/client.pem  --key-file /etc/etcd-certs/client-key.pem -ca-file /etc/etcd-certs/ca.pem --endpoints https://192.168.56.132:2379 get foo
```


## one node
```=
etcd --name test_01 --data-dir /root/etcd-dir --listen-client-urls https://192.168.254.129:2379,https://127.0.0.1:2379 --advertise-client-urls https://192.168.254.129:2379 --ca-file=/etc/etcd-certs/ca.pem --client-cert-auth --trusted-ca-file=/etc/etcd-certs/ca.pem --cert-file=/etc/etcd-certs/server.pem --key-file=/etc/etcd-certs/server-key.pem
etcdctl --cert-file /etc/etcd-certs/client.pem --key-file /etc/etcd-certs/client-key.pem -ca-file /etc/etcd-certs/ca.pem -endpoints https://192.168.254.129:2379 cluster-health


```