Etcd高可用集群部署

搭建高可用 etcd 集群, 高可用的 etcd 正是高可用 k8s 集群的必备基石, 带你一步步生成 etcd 证书, 到 etcd 集群的部署与生产化运维实践

爱菲尔的梦

554人浏览 · 2023-12-27 21:33:29

爱菲尔的梦 · 2023-12-27 21:33:29 发布

在这里插入图片描述

本文来自我的博客地址

主机规划

主机	IP
node1	11.0.1.144
node2	11.0.1.145
node3	11.0.1.146

确保 Go 版本 1.15+

etcd 集群证书

安装 cfssl

apt install golang-cfssl -y

生成默认证书

cfssl print-defaults config > ca-config.json
cfssl print-defaults crs > ca-crs.json

修改证书

root@ubuntu:~/tls# cat ca-config.json
{
    "signing": {
        "default": {
            "expiry": "876000h"
        },
        "profiles": {
            "etcdha": {
                "expiry": "876000h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth",
                    "server auth"
                ]
            }
        }
    }
}

root@ubuntu:~/tls# cat ca-csr.json
{
    "CN": "CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "O": "etcd",
            "ST": "xian",
            "L": "chengdu",
            "OU": "system"
        }
    ]
}

生成 CA 证书

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

创建 etcd 证书签名请求（etcd-csr.json)

root@ubuntu:~/tls# cat etcd-csr.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "11.0.1.144",
    "11.0.1.145",
    "11.0.1.146"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "O": "etcdha"
    }
  ]
}

使用 ca 证书签发 etcd 证书

cfssl gencert -ca ca.pem -ca-key ca-key.pem -config ca-config.json -profile etcdha etcd-csr.json | cfssljson -bare etcd

root@ubuntu:~/tls# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem  etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem

复制证书

mkdir /opt/etcd -p
cp etcd*.pem ca*.pem /opt/etcd/ssl/
cd /opt/etcd/ssl

# 确保事先有相应目录
scp *.pem root@11.0.1.145:/opt/etcd/ssl/
scp *.pem root@11.0.1.146:/opt/etcd/ssl/

etcd HA 部署

安装 etcd

ETCD_VER=v3.5.11

# choose either URL
GOOGLE_URL=https://storage.googleapis.com/etcd
GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
DOWNLOAD_URL=${GOOGLE_URL}

rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test

curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz

/tmp/etcd-download-test/etcd --version
/tmp/etcd-download-test/etcdctl version
/tmp/etcd-download-test/etcdutl version

mv /tmp/etcd-download-test/etcd /bin/
mv /tmp/etcd-download-test/etcdctl /bin/
mv /tmp/etcd-download-test/etcdutl /bin/

分别在各个节点启动 etcd

node1:

nohup etcd --name infra0 \
--data-dir=/tmp/etcd/infra0 \
--listen-peer-urls https://11.0.1.144:2380 \
--initial-advertise-peer-urls https://11.0.1.144:2380 \
--listen-client-urls https://11.0.1.144:2379 \
--advertise-client-urls https://11.0.1.144:2379 \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2380,infra2=https://11.0.1.146:2380 \
--initial-cluster-state new \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra0.log &

node2:

nohup etcd --name infra1 \
--data-dir=/tmp/etcd/infra1 \
--listen-peer-urls https://11.0.1.145:2380 \
--initial-advertise-peer-urls https://11.0.1.145:2380 \
--listen-client-urls https://11.0.1.145:2379 \
--advertise-client-urls https://11.0.1.145:2379 \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2380,infra2=https://11.0.1.146:2380 \
--initial-cluster-state new \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra1.log &

node3:

nohup etcd --name infra2 \
--data-dir=/tmp/etcd/infra2 \
--listen-peer-urls https://11.0.1.146:2380 \
--initial-advertise-peer-urls https://11.0.1.146:2380 \
--listen-client-urls https://11.0.1.146:2379 \
--advertise-client-urls https://11.0.1.146:2379 \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2380,infra2=https://11.0.1.146:2380 \
--initial-cluster-state new \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra2.log &

查看集群状态

root@ubuntu:/opt/etcd/ssl# etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  member list -wtable
+------------------+---------+--------+-------------------------+-------------------------+------------+
|        ID        | STATUS  |  NAME  |       PEER ADDRS        |      CLIENT ADDRS       | IS LEARNER |
+------------------+---------+--------+-------------------------+-------------------------+------------+
|  68e936a39e332ee | started | infra1 | https://11.0.1.145:2380 | https://11.0.1.145:2379 |      false |
| 65d6166ada62ab60 | started | infra0 | https://11.0.1.144:2380 | https://11.0.1.144:2379 |      false |
| 8a036567f69e370a | started | infra2 | https://11.0.1.146:2380 | https://11.0.1.146:2379 |      false |
+------------------+---------+--------+-------------------------+-------------------------+------------+

etcd 集群备份与恢复

插入数据

root@ubuntu:~# etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  put /test1 val1
OK
root@ubuntu:~# etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  put /test2 val2
OK
root@ubuntu:~# etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  put /test1/t1 val1/v1
OK

root@ubuntu:~# etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  get --prefix /test
/test1
val1
/test1/t1
val1/v1
/test2
val2

备份 etcd 数据

etcdctl --endpoints=https://11.0.1.144:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  snapshot save /tmp/snapshot.db

scp /tmp/snapshot.db root@11.0.1.145:/tmp/
scp /tmp/snapshot.db root@11.0.1.146:/tmp/

杀掉个节点 etcd 进程

ps -ef | grep infra | grep -v grep | awk '{print $2}' | xargs kill -9
ps -ef | grep etcd

删除 etcd 数据

rm -rf /tmp/etcd

恢复 etcd 数据

node1:

export ETCDCTL_API=3
etcdctl snapshot restore /tmp/snapshot.db \
  --name infra0 \
  --data-dir=/tmp/etcd/infra0 \
  --initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2379,infra2=https://11.0.1.146:2379 \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls https://11.0.1.144:2380

node2:

export ETCDCTL_API=3
etcdctl snapshot restore /tmp/snapshot.db \
    --name infra1 \
    --data-dir=/tmp/etcd/infra1 \
    --initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2380,infra2=https://11.0.1.146:2380 \
    --initial-cluster-token etcd-cluster-1 \
    --initial-advertise-peer-urls https://11.0.1.145:2380

node3:

export ETCDCTL_API=3
etcdctl snapshot restore /tmp/snapshot.db \
  --name infra2 \
  --data-dir=/tmp/etcd/infra2 \
  --initial-cluster infra0=https://11.0.1.144:2380,infra1=https://11.0.1.145:2380,infra2=https://11.0.1.146:2380 \
  --initial-cluster-token etcd-cluster-1 \
  --initial-advertise-peer-urls https://11.0.1.146:2380

重启 etcd

node1:

nohup etcd --name infra0 \
--data-dir=/tmp/etcd/infra0 \
--listen-peer-urls https://11.0.1.144:2380 \
--listen-client-urls https://11.0.1.144:2379 \
--advertise-client-urls https://11.0.1.144:2379 \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra0.log &

node2:

nohup etcd --name infra1 \
--data-dir=/tmp/etcd/infra1 \
--listen-peer-urls https://11.0.1.145:2380 \
--listen-client-urls https://11.0.1.145:2379 \
--advertise-client-urls https://11.0.1.145:2379 \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra1.log &

node3:

nohup etcd --name infra2 \
--data-dir=/tmp/etcd/infra2 \
--listen-peer-urls https://11.0.1.146:2380 \
--listen-client-urls https://11.0.1.146:2379 \
--advertise-client-urls https://11.0.1.146:2379 \
--client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem 2>&1 > /var/log/infra2.log &

查看集群状态

root@ubuntu:/tmp# etcdctl --endpoints=https://11.0.1.146:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  member list -wtable
+------------------+-----------+--------+-------------------------+-------------------------+------------+
|        ID        |  STATUS   |  NAME  |       PEER ADDRS        |      CLIENT ADDRS       | IS LEARNER |
+------------------+-----------+--------+-------------------------+-------------------------+------------+
|  68e936a39e332ee |   started | infra1 | https://11.0.1.145:2380 | https://11.0.1.145:2379 |      false |
| 65d6166ada62ab60 | unstarted |        | https://11.0.1.144:2380 |                         |      false |
| 8a036567f69e370a |   started | infra2 | https://11.0.1.146:2380 | https://11.0.1.146:2379 |      false |
+------------------+-----------+--------+-------------------------+-------------------------+------------+
root@ubuntu:/tmp# etcdctl --endpoints=https://11.0.1.146:2379  --cacert /opt/etcd/ssl/ca.pem --cert /opt/etcd/ssl/etcd.pem --key /opt/etcd/ssl/etcd-key.pem  get --prefix /
/test1
val1
/test1/t1
val1/v1
/test2
val2

infra0未启动成功, 其余两个节点正常, 原因未知, 待大佬指正

参考:

etcd使用Cfssl生成自签证书(pem) - 西瓜君~ - 博客园 (cnblogs.com)
[svc]证书各个字段的含义 - _毛台 - 博客园 (cnblogs.com)

K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 | 韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。长夜有穷，真水无香。领略过了 K8s“身在江

K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub