二进制k8s集群etcd备份和恢复

当前环境

##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.203 node03
192.168.48.54  nfs

## keepalived的vip
192.168.48.66

IP	Hostname	CPU	Memory
192.168.48.101	master01	2	4G
192.168.48.102	master02	2	4G
192.168.48.103	master03	2	4G
192.168.48.201	node01	2	4G
192.168.48.202	node02	2	4G
192.168.48.203	node03	2	4G
192.168.48.54	nfs	2	4G

软件	版本
kubernetes	1.15.3
docker-ce	19.03
calico	3.8
etcd	3.3.13
CNI	0.8.1
coredns	1.4.0

当前版本

[root@master01 ~]# kubectl get node
NAME       STATUS   ROLES    AGE   VERSION
master01   Ready    master   29d   v1.15.3
master02   Ready    master   29d   v1.15.3
master03   Ready    master   29d   v1.15.3
node01     Ready    node     29d   v1.15.3
node02     Ready    node     29d   v1.15.3
node03     Ready    node     77m   v1.15.3

etcd操作

• –keys-only 默认为true,只显示key,如果设置为false,会显示key的所有值.
• –prefix 默认为true可以看到所有的子目录.

查看版本

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  version
etcdctl version: 3.3.13
API version: 3.3

查看etcd集群

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  member list -w table
+------------------+---------+----------+-----------------------------+-----------------------------+
|        ID        | STATUS  |   NAME   |         PEER ADDRS          |        CLIENT ADDRS         |
+------------------+---------+----------+-----------------------------+-----------------------------+
|  cf07d604d88be6a | started | master01 | https://192.168.48.101:2380 | https://192.168.48.101:2379 |
| 6c8995c4a94f5a29 | started | master03 | https://192.168.48.103:2380 | https://192.168.48.103:2379 |
| 8cf70d11a9c8d0c5 | started | master02 | https://192.168.48.102:2380 | https://192.168.48.102:2379 |
+------------------+---------+----------+-----------------------------+-----------------------------+

查看pod资源

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  get /registry/pods --prefix --keys-only 
/registry/pods/default/counter

/registry/pods/kube-system/calico-kube-controllers-7bd78b474d-r2tdd

/registry/pods/kube-system/calico-node-cfckb

/registry/pods/kube-system/calico-node-hhtlb

/registry/pods/kube-system/calico-node-kzqg6

/registry/pods/kube-system/calico-node-msmv2

/registry/pods/kube-system/calico-node-p2kvc

/registry/pods/kube-system/calico-node-q8tr4

/registry/pods/kube-system/coredns-5c6c9cf6c8-ppmf4

/registry/pods/kube-system/coredns-5c6c9cf6c8-x2sj2

查看service资源

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  get /registry/services --prefix --keys-only
/registry/services/endpoints/default/kubernetes

/registry/services/endpoints/default/nfs-k8s

/registry/services/endpoints/kube-system/kube-controller-manager

/registry/services/endpoints/kube-system/kube-dns

/registry/services/endpoints/kube-system/kube-scheduler

/registry/services/specs/default/kubernetes

/registry/services/specs/kube-system/kube-dns

删除操作

[root@master01 ~]# kubectl get pod
NAME      READY   STATUS    RESTARTS   AGE
counter   1/1     Running   1          10d

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  del /registry/pods/default/counter
1

[root@master01 ~]# kubectl get pod
No resources found.

备份etcd

[root@master01 etcd]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt  snapshot save /etc/etcd/20190911backup.db
Snapshot saved at /etc/etcd/20190911backup.db

[root@master01 etcd]# cd /etc/etcd/
[root@master01 etcd]# ll
total 3812
-rw-r--r-- 1 root root 3891232 Sep 11 23:09 20190911backup.db
-rw-r--r-- 1 root root    1506 Aug 12 22:02 config.yml
-rw-r--r-- 1 root root    1516 Aug 12 22:07 etcd.config.yml

恢复etcd

关闭kube-apiserver

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  ssh ${MasterArray[$name]} "systemctl stop kube-apiserver.service"
done

关闭etcd集群

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  ssh ${MasterArray[$name]} "systemctl stop etcd.service"
done

删除etcd集群的数据

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  ssh ${MasterArray[$name]} "rm -rf /var/lib/etcd/"
done

将etcd备份数据发到etcd节点

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  scp /etc/etcd/20190911backup.db ${MasterArray[$name]}:/etc/etcd/20190911backup.db
done

恢复etcd

[root@master01 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore  /etc/etcd/20190911backup.db

[root@master02 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore  /etc/etcd/20190911backup.db

[root@master03 ~]# ETCDCTL_API=3 etcdctl --endpoints="https://192.168.48.101:2379,https://192.168.48.102:2379,https://192.168.48.103:2379" --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key --cacert=/etc/kubernetes/pki/etcd/ca.crt --data-dir=/var/lib/etcd snapshot restore  /etc/etcd/20190911backup.db

启动etcd

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  ssh ${MasterArray[$name]} "systemctl start etcd.service"
done

启动kube-apiserver

for name in ${!MasterArray[@]};do 
      echo "--- $name ${MasterArray[$name]} ---"
  ssh ${MasterArray[$name]} "systemctl start kube-apiserver.service"
done