一.环境说明

一.环境说明

操作系统：centos7
kubernetes：16.0
docker：18.06

主机名	IP地址	类型
root@K8S-MASTER-ETCD01	192.168.1.121	masters
root@K8S-MASTER-ETCD01	192.168.1.122	masters
root@K8S-MASTER-ETCD01	192.168.1.123	masters
root@K8S-NODE01	192.168.1.124	nodes
root@K8S-NODE01	192.168.1.125	nodes
JENKINS- NGINX01	192.168.1.181	nodes
GITLAB-NGINX01	192.168.1.182	nodes
vip	192.168.1.180	vips

以下操作若无特别说明，默认在master-1上执行

二.环境检查

2.1.免密码登录，安装ansible(yum -y install ansible)

[root@K8S-MASTER-ETCD01 ~]# ansible all -m ping 
master02 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
etcd02 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
etcd03 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
master01 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
etcd01 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
master03 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
nginx02 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
jenkins | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
gitlab | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
nginx01 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
node01 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}
node02 | SUCCESS => {
    "changed": false, 
    "ping": "pong"
}

2.2.添加hosts文件解析

[root@K8S-MASTER-ETCD01 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.121 k8s-master01 k8s-etcd01
192.168.1.122 k8s-master02 k8s-etcd02
192.168.1.123 k8s-master03 k8s-etcd03
192.168.1.124 k8s-node01
192.168.1.125 k8s-node02
192.168.1.181 jenkins-nginx01
192.168.1.182 gitlab-nginx02

[root@K8S-MASTER-ETCD01 ~]# ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts"
master01 | SUCCESS => {
    "changed": false, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "gid": 0, 
    "group": "root", 
    "mode": "0644", 
    "owner": "root", 
    "path": "/etc/hosts", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "state": "file", 
    "uid": 0
}
node01 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814043.1-73523739693166/source", 
    "state": "file", 
    "uid": 0
}
master02 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814043.19-47629036901499/source", 
    "state": "file", 
    "uid": 0
}
node02 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814043.16-130159383122127/source", 
    "state": "file", 
    "uid": 0
}
master03 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814043.22-34117441733976/source", 
    "state": "file", 
    "uid": 0
}
nginx01 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814045.06-94650869788253/source", 
    "state": "file", 
    "uid": 0
}
jenkins | SUCCESS => {
    "changed": false, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "secontext": "system_u:object_r:net_conf_t:s0", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814045.53-139340521245766/source", 
    "state": "file", 
    "uid": 0
}
nginx02 | SUCCESS => {
    "changed": true, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814045.48-244060477753443/source", 
    "state": "file", 
    "uid": 0
}
gitlab | SUCCESS => {
    "changed": false, 
    "checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4", 
    "dest": "/etc/hosts", 
    "gid": 0, 
    "group": "root", 
    "md5sum": "5c1d336967caaa55800c476a026c98f0", 
    "mode": "0644", 
    "owner": "root", 
    "size": 381, 
    "src": "/root/.ansible/tmp/ansible-tmp-1575814045.56-202915694332099/source", 
    "state": "file", 
    "uid": 0
}

2.3.关闭防火墙、selinux、swap分区

[root@K8S-MASTER-ETCD01 ~]# ansible all -m shell -a "systemctl stop firewalld && systemctl disable firewalld && swapoff -a && setenforce 0"    
node01 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

node02 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

master03 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

master02 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

master01 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.

nginx01 | SUCCESS | rc=0 >>


jenkins | SUCCESS | rc=0 >>

[root@K8S-MASTER-ETCD01 ~]# ansible all -m  shell -a  "sed -i '/swap/ s/^/#/' /etc/fstab && swapoff -a"
[root@K8S-MASTER-ETCD01 ~]# ansible all -m  shell -a "sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config"

三.创建etcd证书

3.1.SSL证书准备

##创建etcd集群的ssl文件存放目录
[root@K8S-MASTER-ETCD01 ~]# mkdir -p /etc/ssl/etcd && cd /etc/ssl/etcd
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json  ca-csr.json  server-csr.json
[root@K8S-MASTER-ETCD01 etcd]# cat ca-config.json 
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}

[root@K8S-MASTER-ETCD01 etcd]# cat ca-csr.json 
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}

[root@K8S-MASTER-ETCD01 etcd]# cat server-csr.json 
{
    "CN": "etcd",
    "hosts": [
        "192.168.1.121",
        "192.168.1.122",
        "192.168.1.123"
        ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}

##生成CA证书
[root@K8S-MASTER-ETCD01 etcd]# ls /usr/bin/cfssl*       
/usr/bin/cfssl  /usr/bin/cfssl-certinfo  /usr/bin/cfssljson
[root@K8S-MASTER-ETCD01 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2019/12/08 22:27:49 [INFO] generating a new CA key and certificate from CSR
2019/12/08 22:27:49 [INFO] generate received request
2019/12/08 22:27:49 [INFO] received CSR
2019/12/08 22:27:49 [INFO] generating key: rsa-2048
2019/12/08 22:27:50 [INFO] encoded CSR
2019/12/08 22:27:50 [INFO] signed certificate with serial number 370071359369157884342803152496145563510507956814
[root@K8S-MASTER-ETCD01 etcd]# echo $?
0
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  server-csr.json
[root@K8S-MASTER-ETCD01 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2019/12/08 22:28:21 [INFO] generate received request
2019/12/08 22:28:21 [INFO] received CSR
2019/12/08 22:28:21 [INFO] generating key: rsa-2048
2019/12/08 22:28:22 [INFO] encoded CSR
2019/12/08 22:28:22 [INFO] signed certificate with serial number 408695704641215627380878767968987768712130082527
2019/12/08 22:28:22 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@K8S-MASTER-ETCD01 etcd]# echo $?
0
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  server.csr  server-csr.json  server-key.pem  server.pem

3.2.将生成的CA证书拷贝到etcd集群的其他主机

[root@K8S-MASTER-ETCD01 etcd]# ansible all -m shell -a "mkdir -p /etc/ssl/etcd/"
node02 | SUCCESS | rc=0 >>
master03 | SUCCESS | rc=0 >>
node01 | SUCCESS | rc=0 >>
master02 | SUCCESS | rc=0 >>
master01 | SUCCESS | rc=0 >>
nginx02 | SUCCESS | rc=0 >>
nginx01 | SUCCESS | rc=0 >>
[root@K8S-MASTER-ETCD01 etcd]# scp *.pem 192.168.1.122:/etc/ssl/etcd/
ca-key.pem                                                                                                                                                                                                                    100% 1679   146.9KB/s   00:00    
ca.pem                                                                                                                                                                                                                        100% 1265     1.5MB/s   00:00    
server-key.pem                                                                                                                                                                                                                100% 1675     1.7MB/s   00:00    
server.pem                                                                                                                                                                                                                    100% 1338     1.6MB/s   00:00    
[root@K8S-MASTER-ETCD01 etcd]# scp *.pem 192.168.1.123:/etc/ssl/etcd/
ca-key.pem                                                                                                                                                                                                                    100% 1679     1.5MB/s   00:00    
ca.pem                                                                                                                                                                                                                        100% 1265     1.2MB/s   00:00    
server-key.pem                                                                                                                                                                                                                100% 1675     1.6MB/s   00:00    
server.pem 

3.3.获取etcd二进制文件

下载路径：https://github.com/etcd-io/etcd/releases，解压出来后，只提取etcdctl、etcd两个文件
[root@K8S-MASTER-ETCD01 etcd]# cd ~
[root@K8S-MASTER-ETCD01 ~]# ls /usr/bin/etcd*
/usr/bin/etcd  /usr/bin/etcdctl
[root@K8S-MASTER-ETCD01 ~]# chmod +x /usr/bin/etcd*
[root@K8S-MASTER-ETCD01 ~]# scp /usr/bin/etcd* 192.168.1.123:/usr/bin/
etcd                                                                                                                                                                                                                          100%   17MB  57.8MB/s   00:00    
etcdctl                                                                                                                                                                                                                       100%   15MB  59.1MB/s   00:00    
[root@K8S-MASTER-ETCD01 ~]# scp /usr/bin/etcd* 192.168.1.122:/usr/bin/
etcd                                                                                                                                                                                                                          100%   17MB  64.3MB/s   00:00    
etcdctl                                                                                                                                                                                                                           100%   15MB  59.1MB/s   00:00    
   

3.4.编写conf配置文件

[root@K8S-MASTER-ETCD01 ~]#  ansible all -m shell -a "mkdir -p /etc/etcd/"
node02 | SUCCESS | rc=0 >>
master02 | SUCCESS | rc=0 >>
node01 | SUCCESS | rc=0 >>
master03 | SUCCESS | rc=0 >>
master01 | SUCCESS | rc=0 >>
nginx02 | SUCCESS | rc=0 >>
nginx01 | SUCCESS | rc=0 >>
[root@K8S-MASTER-ETCD01 ~]# cd /etc/etcd/
[root@K8S-MASTER-ETCD01 etcd]# cat /etc/etcd/etcd.conf 

#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.121:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.121:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.121:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.121:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

[root@K8S-MASTER-ETCD02 ~]# cat /etc/etcd/etcd.conf

#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.122:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.122:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.122:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.122:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

##
[root@K8S-MASTER-ETCD03 ~]# cat /etc/etcd/etcd.conf

#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.123:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.123:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.123:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.123:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

3.5.编辑etcd.service文件

[root@K8S-MASTER-ETCD01 ~]# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
        --name=${ETCD_NAME} \
        --data-dir=${ETCD_DATA_DIR} \
        --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
        --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
        --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
        --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
        --initial-cluster=${ETCD_INITIAL_CLUSTER} \
        --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
        --initial-cluster-state=new \
        --cert-file=/etc/ssl/etcd/server.pem \
        --key-file=/etc/ssl/etcd/server-key.pem \
        --peer-cert-file=/etc/ssl/etcd/server.pem \
        --peer-key-file=/etc/ssl/etcd/server-key.pem \
        --trusted-ca-file=/etc/ssl/etcd/ca.pem \
        --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

[root@K8S-MASTER-ETCD02 ~]# cat /usr/lib/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
        --name=${ETCD_NAME} \
        --data-dir=${ETCD_DATA_DIR} \
        --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
        --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
        --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
        --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
        --initial-cluster=${ETCD_INITIAL_CLUSTER} \
        --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
        --initial-cluster-state=new \
        --cert-file=/etc/ssl/etcd/server.pem \
        --key-file=/etc/ssl/etcd/server-key.pem \
        --peer-cert-file=/etc/ssl/etcd/server.pem \
        --peer-key-file=/etc/ssl/etcd/server-key.pem \
        --trusted-ca-file=/etc/ssl/etcd/ca.pem \
        --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
[root@K8S-MASTER-ETCD03 ~]# cat /usr/lib/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
        --name=${ETCD_NAME} \
        --data-dir=${ETCD_DATA_DIR} \
        --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
        --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
        --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
        --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
        --initial-cluster=${ETCD_INITIAL_CLUSTER} \
        --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
        --initial-cluster-state=new \
        --cert-file=/etc/ssl/etcd/server.pem \
        --key-file=/etc/ssl/etcd/server-key.pem \
        --peer-cert-file=/etc/ssl/etcd/server.pem \
        --peer-key-file=/etc/ssl/etcd/server-key.pem \
        --trusted-ca-file=/etc/ssl/etcd/ca.pem \
        --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

3.6.启动etcd集群

[root@K8S-MASTER-ETCD01 ~]# ansible masters -m shell -a " systemctl start etcd &&systemctl enable etcd && systemctl status etcd  "
## 验证是否为开机启动
[root@K8S-MASTER-ETCD01 ~]# systemctl list-unit-files |grep etcd
etcd.service                                  enabled

四.验证etcd集群是否搭建成功

[root@K8S-MASTER-ETCD01 ~]# etcdctl  --ca-file=/etc/ssl/etcd/ca.pem --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --endpoints=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 cluster-health
member 140f63cd51ff8859 is healthy: got healthy result from https://192.168.1.121:2379
member b5900dafc7a3b5f9 is healthy: got healthy result from https://192.168.1.122:2379
member c48307bcc0ac155e is healthy: got healthy result from https://192.168.1.123:2379
cluster is healthy