系统版本
[root@k8s-master1 k8s]# cat /etc/redhat-release 
CentOS Linux release 7.4.1708 (Core) 
 
docker版本
[root@k8s-master1 ~]#  docker -v
Docker version 20.10.15, build fd82621
 
k8s版本
[root@k8s-master1 ~]# kubelet --version
Kubernetes v1.23.6
[root@k8s-master1 ~]# kubectl version
Client Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.6", GitCommit:"ad3338546da947756e8a88aa6822e9c11e7eac22", GitTreeState:"clean", BuildDate:"2022-04-14T08:49:13Z", GoVersion:"go1.17.9", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.6", GitCommit:"ad3338546da947756e8a88aa6822e9c11e7eac22", GitTreeState:"clean", BuildDate:"2022-04-14T08:43:11Z", GoVersion:"go1.17.9", Compiler:"gc", Platform:"linux/amd64"}
[root@k8s-master1 ~]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.6", GitCommit:"ad3338546da947756e8a88aa6822e9c11e7eac22", GitTreeState:"clean", BuildDate:"2022-04-14T08:48:05Z", GoVersion:"go1.17.9", Compiler:"gc", Platform:"linux/amd64"}
 
 
1、ip规划
k8s-master1 : 192.168.73.130
k8s-master2 : 192.168.73.131
k8s-master3 : 192.168.73.132
k8s-node1 : 192.168.73.133
k8s-node2 : 192.168.73.134
 
2、ip配置
nmcli connection modify ens33 ipv4.method manual ipv4.address 192.168.73.130/24  ipv4.gateway 192.168.73.2 connection.autoconnect yes
nmcli connection up ens33
 
nmcli connection modify ens33 ipv4.method manual ipv4.address 192.168.73.131/24  ipv4.gateway 192.168.73.2 connection.autoconnect yes
nmcli connection up ens33
 
nmcli connection modify ens33 ipv4.method manual ipv4.address 192.168.73.132/24  ipv4.gateway 192.168.73.2 connection.autoconnect yes
nmcli connection up ens33
 
nmcli connection modify ens33 ipv4.method manual ipv4.address 192.168.73.133/24  ipv4.gateway 192.168.73.2 connection.autoconnect yes
nmcli connection up ens33
 
nmcli connection modify ens33 ipv4.method manual ipv4.address 192.168.73.134/24  ipv4.gateway 192.168.73.2 connection.autoconnect yes
nmcli connection up ens33
 
 
 
 
 
1、关闭firewalld和selinux（所有主机）
vi /etc/selinux/config
 
getenforce 
setenforce 0
getenforce 
 
systemctl stop firewalld
 systemctl disable firewalld
 
2、配置解析/etc/hosts（所有主机）及主机名
hostnamectl set-hostname k8s-master1
hostnamectl set-hostname k8s-master2
hostnamectl set-hostname k8s-master3
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2
 
vim /etc/hosts
192.168.73.130 k8s-master1
192.168.73.131 k8s-master2
192.168.73.132 k8s-master3
192.168.73.133 k8s-node1
192.168.73.134 k8s-node2
 
将文件传入其他主机
for i in {131..134};do scp /etc/hosts 192.168.73.$i:/etc/hosts ;done
 
3、添加内核参数文件 /etc/sysctl.d/k8s.conf（所有主机）
vi /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
 
 
 
4、执行命令（所有主机）
modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf
 
 

5、各种安装包的查找及安装
配置清华镜像源作为yum源，下载各种软件包
基本包的镜像源
/etc/yum.repos.d/qinghua-base.repo

[base]
name=CentOS-$releasever - Base
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/os/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
enabled=1
gpgcheck=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-7
 
#released updates
[updates]
name=CentOS-$releasever - Updates
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/updates/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates
enabled=1
gpgcheck=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-7
 
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/extras/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras
enabled=1
gpgcheck=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-7
 
#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/centosplus/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus
gpgcheck=0
enabled=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-7


6、k8s镜像源
/etc/yum.repos.d/k8s-base.repo

[k8s-base]
name=k8s-base
baseurl=https://mirrors.tuna.tsinghua.edu.cn/kubernetes/yum/repos/kubernetes-el7-x86_64/
ebabled=1
gpgcheck=0


7、docker镜像源
/etc/yum.repos.d/docker-base.repo

[docker-base]
name=docker-base
baseurl=https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/centos/7/x86_64/stable/
ebabled=1
gpgcheck=0


8、查询k8s相关软件的版本
[root@localhost yum.repos.d]# yum list |grep kubeadm
kubeadm.x86_64                              1.23.6-0                   k8s-base 
[root@localhost yum.repos.d]# yum list |grep kubectl
kubectl.x86_64                              1.23.6-0                   k8s-base 
[root@localhost yum.repos.d]# 
[root@localhost yum.repos.d]# yum list |grep kubelet
kubelet.x86_64                              1.23.6-0                   k8s-base 

9、下载相关的k8s软件包
[root@localhost yum.repos.d]# mkdir -p /data/bao/k8s
yum -y install --downloadonly --downloaddir=/data/bao/k8s kubeadm kubelet kubectl

10、查看下载的软件包
[root@localhost k8s]# ll /data/bao/k8s/
总用量 66812
-rw-r--r--. 1 root root  7401938 3月  17 08:32 4d300a7655f56307d35f127d99dc192b6aa4997f322234e754f16aaa60fd8906-cri-tools-1.23.0-0.x86_64.rpm
-rw-r--r--. 1 root root 21556178 4月  21 06:23 68a98b2ae673eef4a5ddbf1f3c830db0df8fbb888e035aea6054677d88f8a8bc-kubelet-1.23.6-0.x86_64.rpm
-rw-r--r--. 1 root root  9921862 4月  21 06:23 868c4a6ee448d1e8488938812a19a991b5132c81de511cd737d93493b98451cc-kubectl-1.23.6-0.x86_64.rpm
-rw-r--r--. 1 root root  9478890 4月  21 12:39 89104c7beafab5f04d6789e5425963fc8f91ba9711c9603f1ad89003cdea4fe4-kubeadm-1.23.6-0.x86_64.rpm
-rw-r--r--. 1 root root   191000 4月   4 2020 conntrack-tools-1.4.4-7.el7.x86_64.rpm
-rw-r--r--. 1 root root 19487362 9月  10 2020 db7cb5cb0b3f6875f54d10f02e625573988e3e91fd4fc5eef0b1876bb18604ad-kubernetes-cni-0.8.7-0.x86_64.rpm
-rw-r--r--. 1 root root    18400 4月   4 2020 libnetfilter_cthelper-1.0.0-11.el7.x86_64.rpm
-rw-r--r--. 1 root root    18212 4月   4 2020 libnetfilter_cttimeout-1.0.0-7.el7.x86_64.rpm
-rw-r--r--. 1 root root    23584 8月  11 2017 libnetfilter_queue-1.0.2-2.el7_2.x86_64.rpm
-rw-r--r--. 1 root root   296632 8月  11 2017 socat-1.7.3.2-2.el7.x86_64.rpm

11、查询docker相关的软件版本
[root@localhost k8s]# yum list |grep  docker-ce
docker-ce.x86_64                            3:20.10.15-3.el7           docker-base
docker-ce-cli.x86_64                        1:20.10.15-3.el7           docker-base
docker-ce-rootless-extras.x86_64            20.10.15-3.el7             docker-base
docker-ce-selinux.noarch                    17.03.3.ce-1.el7           docker-base

12、下载相关的docker软件包
[root@localhost k8s]# mkdir -p /data/bao/docker
yum -y install --downloadonly --downloaddir=/data/bao/docker docker-ce
yum -y install --downloadonly --downloaddir=/data/bao/docker libcgroup fuse3-libs

13、查看下载的相关软件包
[root@localhost docker]# ll /data/bao/docker/
总用量 110812
-rw-r--r--. 1 root root   261632 8月  23 2019 audit-2.8.5-4.el7.x86_64.rpm
-rw-r--r--. 1 root root   104408 8月  23 2019 audit-libs-2.8.5-4.el7.x86_64.rpm
-rw-r--r--. 1 root root    78256 8月  23 2019 audit-libs-python-2.8.5-4.el7.x86_64.rpm
-rw-r--r--. 1 root root   302068 11月 12 2018 checkpolicy-2.5-8.el7.x86_64.rpm
-rw-r--r--. 1 root root 34647136 5月   6 05:05 containerd.io-1.6.4-3.1.el7.x86_64.rpm
-rw-r--r--. 1 root root    40816 7月   6 2020 container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm
-rw-r--r--. 1 root root 23234292 5月   6 05:05 docker-ce-20.10.15-3.el7.x86_64.rpm
-rw-r--r--. 1 root root 30837648 5月   6 05:05 docker-ce-cli-20.10.15-3.el7.x86_64.rpm
-rw-r--r--. 1 root root  8638396 5月   6 05:05 docker-ce-rootless-extras-20.10.15-3.el7.x86_64.rpm
-rw-r--r--. 1 root root  3930044 3月  11 00:47 docker-scan-plugin-0.17.0-3.el7.x86_64.rpm
-rw-r--r--. 1 root root    83764 4月  29 2020 fuse3-libs-3.6.1-4.el7.x86_64.rpm
-rw-r--r--. 1 root root    55796 4月  29 2020 fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm
-rw-r--r--. 1 root root    67720 8月  23 2019 libcgroup-0.41-21.el7.x86_64.rpm
-rw-r--r--. 1 root root   166012 4月   4 2020 libselinux-2.5-15.el7.x86_64.rpm
-rw-r--r--. 1 root root   241168 4月   4 2020 libselinux-python-2.5-15.el7.x86_64.rpm
-rw-r--r--. 1 root root   154876 4月   4 2020 libselinux-utils-2.5-15.el7.x86_64.rpm
-rw-r--r--. 1 root root   154244 11月 12 2018 libsemanage-2.5-14.el7.x86_64.rpm
-rw-r--r--. 1 root root   115284 11月 12 2018 libsemanage-python-2.5-14.el7.x86_64.rpm
-rw-r--r--. 1 root root   304196 11月 12 2018 libsepol-2.5-10.el7.x86_64.rpm
-rw-r--r--. 1 root root   938736 4月   4 2020 policycoreutils-2.5-34.el7.x86_64.rpm
-rw-r--r--. 1 root root   468316 4月   4 2020 policycoreutils-python-2.5-34.el7.x86_64.rpm
-rw-r--r--. 1 root root    32880 7月   4 2014 python-IPy-0.75-6.el7.noarch.rpm
-rw-r--r--. 1 root root   509568 11月 18 2020 selinux-policy-3.13.1-268.el7_9.2.noarch.rpm
-rw-r--r--. 1 root root  7335504 11月 18 2020 selinux-policy-targeted-3.13.1-268.el7_9.2.noarch.rpm
-rw-r--r--. 1 root root   635184 11月 12 2018 setools-libs-3.3.8-4.el7.x86_64.rpm
-rw-r--r--. 1 root root    83452 4月  29 2020 slirp4netns-0.4.3-4.el7_8.x86_64.rpm

14、安装依赖包
yum install -y yum-utils device-mapper-persistent-data lvm2
yum install -y epel-release conntrack ipvsadm ipset jq sysstat curl iptables libseccomp
 
15、安装docker
[root@localhost k8s]# cd /data/bao/docker/
[root@localhost docker]# yum -y install *

16、查看docker版本
[root@localhost docker]# docker -v 
Docker version 20.10.15, build fd82621

17、安装k8s相关插件
[root@localhost docker]# cd /data/bao/k8s/
[root@localhost k8s]# yum -y install *

18、查看k8s相关插件的版本
[root@localhost k8s]# kubelet --version
Kubernetes v1.23.6
[root@localhost k8s]# kubectl version
Client Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.6", GitCommit:"ad3338546da947756e8a88aa6822e9c11e7eac22", GitTreeState:"clean", BuildDate:"2022-04-14T08:49:13Z", GoVersion:"go1.17.9", Compiler:"gc", Platform:"linux/amd64"}
The connection to the server localhost:8080 was refused - did you specify the right host or port?
[root@localhost k8s]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.6", GitCommit:"ad3338546da947756e8a88aa6822e9c11e7eac22", GitTreeState:"clean", BuildDate:"2022-04-14T08:48:05Z", GoVersion:"go1.17.9", Compiler:"gc", Platform:"linux/amd64"}

19、设置开机启动并启动相关的软件
systemctl enable docker kubelet
systemctl restart docker

20、查看docker状态
[root@localhost k8s]# systemctl status docker -l
● docker.service - Docker Application Container Engine
   Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled)
   Active: active (running) since 二 2022-05-10 15:57:00 CST; 2min 6s ago
     Docs: https://docs.docker.com
 Main PID: 9862 (dockerd)
   Memory: 37.5M
   CGroup: /system.slice/docker.service
           └─9862 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.025887674+08:00" level=info msg="scheme \"unix\" not registered, fallback to default scheme" module=grpc
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.025911795+08:00" level=info msg="ccResolverWrapper: sending update to cc: {[{unix:///run/containerd/containerd.sock  <nil> 0 <nil>}] <nil> <nil>}" module=grpc
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.025920488+08:00" level=info msg="ClientConn switching balancer to \"pick_first\"" module=grpc
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.054959588+08:00" level=info msg="Loading containers: start."
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.238876078+08:00" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.313376634+08:00" level=info msg="Loading containers: done."
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.370410721+08:00" level=info msg="Docker daemon" commit=4433bf6 graphdriver(s)=overlay2 version=20.10.15
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.370760686+08:00" level=info msg="Daemon has completed initialization"
5月 10 15:57:00 k8s-node1 systemd[1]: Started Docker Application Container Engine.
5月 10 15:57:00 k8s-node1 dockerd[9862]: time="2022-05-10T15:57:00.418165781+08:00" level=info msg="API listen on /var/run/docker.sock"



各种镜像的查找及安装

21、查看创建集群需要的镜像版本
[root@localhost k8s]# kubeadm config images list
I0510 16:01:13.265348   10209 version.go:255] remote version is much newer: v1.24.0; falling back to: stable-1.23
k8s.gcr.io/kube-apiserver:v1.23.6
k8s.gcr.io/kube-controller-manager:v1.23.6
k8s.gcr.io/kube-scheduler:v1.23.6
k8s.gcr.io/kube-proxy:v1.23.6
k8s.gcr.io/pause:3.6
k8s.gcr.io/etcd:3.5.1-0
k8s.gcr.io/coredns/coredns:v1.8.6

22、国内拉取镜像
kubeadm config images pull --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.23.6

或者直接指定拉取镜像(使用，若使用其他方式拉取，镜像标签按需求更改)
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.23.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.23.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.23.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.23.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.8.6
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.1-0
 
23、镜像打标签，删除不用镜像
docker tag 	 registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.6	 k8s.gcr.io/kube-apiserver:v1.23.6
docker tag 	 registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.6 k8s.gcr.io/kube-scheduler:v1.23.6
docker tag 	 registry.aliyuncs.com/google_containers/kube-proxy:v1.23.6	 k8s.gcr.io/kube-proxy:v1.23.6
docker tag 	 registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.6	 k8s.gcr.io/kube-controller-manager:v1.23.6
docker tag 	 registry.aliyuncs.com/google_containers/coredns:v1.8.6	 k8s.gcr.io/coredns/coredns:v1.8.6 
docker tag 	 registry.aliyuncs.com/google_containers/pause:3.6	 k8s.gcr.io/pause:3.6  
docker tag 	 registry.aliyuncs.com/google_containers/etcd:3.5.1-0	 k8s.gcr.io/etcd:3.5.1-0  
 
docker rmi registry.aliyuncs.com/google_containers/kube-proxy:v1.23.6
docker rmi registry.aliyuncs.com/google_containers/kube-controller-manager:v1.23.6
docker rmi registry.aliyuncs.com/google_containers/kube-apiserver:v1.23.6
docker rmi registry.aliyuncs.com/google_containers/kube-scheduler:v1.23.6
docker rmi registry.aliyuncs.com/google_containers/pause:3.6
docker rmi registry.aliyuncs.com/google_containers/coredns:v1.8.6
docker rmi registry.aliyuncs.com/google_containers/etcd:3.5.1-0
 
24、打包镜像，传入其他主机
mkdir -p /data/bao/k8s-images;cd  /data/bao/k8s-images 
docker save k8s.gcr.io/kube-apiserver:v1.23.6 > kube-apiserver.tar
docker save k8s.gcr.io/kube-scheduler:v1.23.6 > kube-scheduler.tar
docker save k8s.gcr.io/kube-controller-manager:v1.23.6 >  kube-controller-manager.tar
docker save k8s.gcr.io/kube-proxy:v1.23.6 > kube-proxy.tar
docker save k8s.gcr.io/coredns/coredns:v1.8.6 > coredns.tar
docker save k8s.gcr.io/pause:3.6 > pause.tar
docker save k8s.gcr.io/etcd:3.5.1-0 > etcd.tar

25、查看打包好的镜像包
[root@localhost k8s-images]# ll /data/bao/k8s-images/
总用量 755464
-rw-r--r--. 1 root root  46967296 5月  10 16:17 coredns.tar
-rw-r--r--. 1 root root 293936128 5月  10 16:17 etcd.tar
-rw-r--r--. 1 root root 136526848 5月  10 16:17 kube-apiserver.tar
-rw-r--r--. 1 root root 126348288 5月  10 16:17 kube-controller-manager.tar
-rw-r--r--. 1 root root 114260480 5月  10 16:17 kube-proxy.tar
-rw-r--r--. 1 root root  54852608 5月  10 16:17 kube-scheduler.tar
-rw-r--r--. 1 root root    692736 5月  10 16:17 pause.tar

26、将所有的安装包及镜像包传入其他主机
for i in {130..132};do scp -r /data 192.168.73.$i:/data ;done

27、其他主机安装docker和k8s相关插件
cd /data/data/bao/docker/ ; yum -y install *
cd /data/data/bao/k8s/ ; yum -y install *

28、设置软件开机自起并启动
systemctl enable docker kubelet
systemctl restart docker

29、将打包好的镜像导入每台主机
cd /data/data/bao/k8s-images
docker load < kube-apiserver.tar
docker load < kube-scheduler.tar
docker load < kube-controller-manager.tar
docker load < kube-proxy.tar
docker load < coredns.tar
docker load < pause.tar
docker load < flannel.tar
docker load < etcd.tar
 
30、查看导入的docker镜像
[root@localhost k8s-images]# docker images
REPOSITORY                           TAG       IMAGE ID       CREATED        SIZE
k8s.gcr.io/kube-apiserver            v1.23.6   8fa62c12256d   3 weeks ago    135MB
k8s.gcr.io/kube-scheduler            v1.23.6   595f327f224a   3 weeks ago    53.5MB
k8s.gcr.io/kube-controller-manager   v1.23.6   df7b72818ad2   3 weeks ago    125MB
k8s.gcr.io/kube-proxy                v1.23.6   4c0375452406   3 weeks ago    112MB
k8s.gcr.io/etcd                      3.5.1-0   25f8c7f3da61   6 months ago   293MB
k8s.gcr.io/coredns/coredns           v1.8.6    a4ca41631cc7   7 months ago   46.8MB
k8s.gcr.io/pause                     3.6       6270bb605e12   8 months ago   683kB


31、、关闭swap（所有主机）
swapoff  -a
sysctl -p /etc/sysctl.d/k8s.conf 
注释掉/etc/fstab中的swap条目
 mount -a
echo "KUBELET_EXTRA_ARGS=--fail-swap-on=false" > /etc/sysconfig/kubelet
 
32、设置开机自启动
systemctl enable kubelet.service docker
systemctl restart docker
systemctl restart kubelet
 
 
33、haproxy部署
k8s-master1节点
 yum install  haproxy -y
 
[root@k8s-node3 yum.repos.d]# cat  /etc/haproxy/haproxy.cfg
# /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    log /dev/log local0
    log /dev/log local1 notice
    daemon
 
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
#defaults
listen stats 0.0.0.0:12345
    mode                    http
    log                     global
	maxconn 10
	stats enable
	stats hide-version
	stats refresh 30s
	stats show-node
	#stats auth admin:p@sssw0rd
	stats uri /stats
	
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 1
    timeout http-request    10s
    timeout queue           20s
    timeout connect         5s
    timeout client          20s
    timeout server          20s
    timeout http-keep-alive 10s
    timeout check           10s
 
#---------------------------------------------------------------------
# apiserver frontend which proxys to the masters
#---------------------------------------------------------------------
frontend apiserver
    bind 0.0.0.0:12567
    mode tcp
    option tcplog
    default_backend kube-api-server
 
#---------------------------------------------------------------------
# round robin balancing for apiserver
#---------------------------------------------------------------------
backend kube-api-server
    option httpchk GET /healthz
    http-check expect status 200
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
    server k8s-master1 192.168.73.130:6443 check
    server k8s-master2 192.168.73.131:6443 check
    server k8s-master3 192.168.73.132:6443 check
 
[root@k8s-node3 yum.repos.d]# systemctl enable haproxy --now
[root@k8s-node3 yum.repos.d]# systemctl restart haproxy
 
 
34、、访问haproxy页面查看k8s集群健康状态
http://192.168.73.130:12345/stats
 
 
35、etcd集群安装
参考 https://blog.csdn.net/liao__ran/article/details/124684090?spm=1001.2014.3001.5502 安装etcd集群
 
36、生成 kubeadm 默认配置文件
[root@localhost k8s-images]# cd /etc/kubernetes/
kubeadm config print init-defaults --component-configs \
KubeProxyConfiguration,KubeletConfiguration > kubeadm-config.yaml

37、k8s集群初始化所需文件，修改后
[root@localhost kubernetes]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.73.130    # 本机IP
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  imagePullPolicy: IfNotPresent
  name: k8s-master1        # 本主机名
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.73.130:12567"    # 虚拟IP和haproxy端口,新增
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  external:
    endpoints:
    - https://192.168.73.130:2379
    - https://192.168.73.131:2379
    - https://192.168.73.132:2379
    caFile: /data/etcd/ssl/ca.pem
    certFile: /data/etcd/ssl/server.pem
    keyFile: /data/etcd/ssl/server-key.pem
imageRepository: k8s.gcr.io	# 镜像仓库源要根据自己实际情况修改
kind: ClusterConfiguration
kubernetesVersion: 1.23.6	# k8s版本
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16	#新增
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
  maxPerCore: null
  min: null
  tcpCloseWaitTimeout: null
  tcpEstablishedTimeout: null
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: null
  minSyncPeriod: 0s
  syncPeriod: 0s
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 0s
  scheduler: ""
  strictARP: false
  syncPeriod: 0s
  tcpFinTimeout: 0s
  tcpTimeout: 0s
  udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: ipvs
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:
  enableDSR: false
  networkName: ""
  sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging:
  flushFrequency: 0
  options:
    json:
      infoBufferSize: "0"
  verbosity: 0
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
 

38、dry-run模式验证语法，验证无报错才能进行初始化操作
kubeadm init --config kubeadm-config.yaml --dry-run




39、集群初始化
初始化日志
 kubeadm init --config=kubeadm-config.yaml   --upload-certs  --ignore-preflight-errors=Swap
报错
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get "http://localhost:10248/healthz": dial tcp [::1]:10248: connect: connection refused.
参考
https://blog.51cto.com/8999a/5005591

解决方案;
所有master跟node节点的docker修改为
[root@k8s-master ~]# cat /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
然后重启docker
systemctl restart docker


40、进行环境重置
kubeadm reset 


41、继续进行初始化
[root@localhost kubernetes]# kubeadm init --config=kubeadm-config.yaml   --upload-certs  --ignore-preflight-errors=Swap
W0511 01:33:54.949759    8830 strict.go:55] error unmarshaling configuration schema.GroupVersionKind{Group:"kubeadm.k8s.io", Version:"v1beta3", Kind:"ClusterConfiguration"}: error unmarshaling JSON: while decoding JSON: json: unknown field "type"
[init] Using Kubernetes version: v1.23.6
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master1 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.73.130]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] External etcd mode: Skipping etcd/ca certificate authority generation
[certs] External etcd mode: Skipping etcd/server certificate generation
[certs] External etcd mode: Skipping etcd/peer certificate generation
[certs] External etcd mode: Skipping etcd/healthcheck-client certificate generation
[certs] External etcd mode: Skipping apiserver-etcd-client certificate generation
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 10.578072 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.23" in namespace kube-system with the configuration for the kubelets in the cluster
NOTE: The "kubelet-config-1.23" naming of the kubelet ConfigMap is deprecated. Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just "kubelet-config". Kubeadm upgrade will handle this transition transparently.
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
5de74b8a94efe7b0dc7f8d27a4310ba69a1d75eedef06981b6461c171798f630
[mark-control-plane] Marking the node k8s-master1 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master1 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b \
	--control-plane --certificate-key 5de74b8a94efe7b0dc7f8d27a4310ba69a1d75eedef06981b6461c171798f630

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b 


 
主节点加入集群
  kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b \
	--control-plane --certificate-key 5de74b8a94efe7b0dc7f8d27a4310ba69a1d75eedef06981b6461c171798f630
 
从节点加入集群
kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b 
 
 
 
42、其他主节点加入集群
k8s-master2加入集群
 
[root@localhost k8s-images]#   kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
> --discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b \
> --control-plane --certificate-key 5de74b8a94efe7b0dc7f8d27a4310ba69a1d75eedef06981b6461c171798f630
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master2 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.73.131 192.168.73.130]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Skipping etcd check in external mode
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[control-plane-join] using external etcd - no local stacked instance added
The 'update-status' phase is deprecated and will be removed in a future release. Currently it performs no operation
[mark-control-plane] Marking the node k8s-master2 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master2 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.


To start administering your cluster from this node, you need to run the following as a regular user:

	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.
 
 
43、k8s-master3加入集群
 
[root@localhost k8s-images]#   kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef \
> --discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b \
> --control-plane --certificate-key 5de74b8a94efe7b0dc7f8d27a4310ba69a1d75eedef06981b6461c171798f630
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master3 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.73.132 192.168.73.130]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Skipping etcd check in external mode
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[control-plane-join] using external etcd - no local stacked instance added
The 'update-status' phase is deprecated and will be removed in a future release. Currently it performs no operation
[mark-control-plane] Marking the node k8s-master3 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master3 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.


To start administering your cluster from this node, you need to run the following as a regular user:

	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.
 
 
 
 
44、node节点加入集群
k8s-node1加入集群
 
[root@localhost data]# kubeadm join 192.168.73.130:12567 --token abcdef.0123456789abcdef  --discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b
[preflight] Running pre-flight checks
	[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
 
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
 
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
 
 

 
45、若证书过期后，可以重新更新证书和token认证，然后再加入

可以参考
https://blog.csdn.net/hunheidaode/article/details/118341134

或者 重新更新证书
[root@k8s-master1 kubernetes]# kubeadm init phase upload-certs --upload-certs
W0803 17:38:07.670046   44778 version.go:102] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": dial tcp: lookup dl.k8s.io on [::1]:53: read udp [::1]:56684->[::1]:53: read: connection refused
W0803 17:38:07.670925   44778 version.go:103] falling back to the local client version: v1.21.3
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
63e54ed5b06bc1f7b93adaeda0df792b12064c11f9d274d9f3d2b5b012cbc584
 
重新生成token认证
[root@k8s-master1 kubernetes]# kubeadm token generate
9psojs.mqrgtud16qjymfok
 

重新加入
kubeadm join 192.168.73.130:12567 --token 0d3mtm.x0o720qncyjl4c0n \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b  --v=5
 
 
或者 创建永久token
 [root@localhost kubernetes]# kubeadm token create --ttl 0
jhdsm4.pq5yxkkw3xa2d765
[root@localhost kubernetes]# kubeadm token list
TOKEN                     TTL         EXPIRES                USAGES                   DESCRIPTION                                                EXTRA GROUPS
2b8fmw.wtv48ygu93kjs4zd   1h          2022-05-10T20:03:20Z   <none>                   Proxy for managing TTL for the kubeadm-certs secret        <none>
5u3uio.x9q1noeofzyibq0r   1h          2022-05-10T19:34:09Z   <none>                   Proxy for managing TTL for the kubeadm-certs secret        <none>
abcdef.0123456789abcdef   23h         2022-05-11T17:34:09Z   authentication,signing   <none>                                                     system:bootstrappers:kubeadm:default-node-token
jhdsm4.pq5yxkkw3xa2d765   <forever>   <never>   authentication,signing   <none>                                                     system:bootstrappers:kubeadm:default-node-token

重新加入
kubeadm join 192.168.73.130:12567 --token 0d3mtm.x0o720qncyjl4c0n \
	--discovery-token-ca-cert-hash sha256:4977075cf76c9e7b7a3a2b8ffa9098cc083fd1bc9d26d414527682bcfbf63e4b  --v=5
 


46、Certificate-key介绍
https://www.kancloud.cn/pshizhsysu/kubernetes/2055662

 
47、K8S节点网络配置(Flanneld)
部署flannel，开网
https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
查看需要的镜像
[root@k8s-node1 data]# cat kube-flannel.yml |grep image
       #image: flannelcni/flannel-cni-plugin:v1.0.1 for ppc64le and mips64le (dockerhub limitations may apply)
        image: rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1
       #image: flannelcni/flannel:v0.17.0 for ppc64le and mips64le (dockerhub limitations may apply)
        image: rancher/mirrored-flannelcni-flannel:v0.17.0
       #image: flannelcni/flannel:v0.17.0 for ppc64le and mips64le (dockerhub limitations may apply)
        image: rancher/mirrored-flannelcni-flannel:v0.17.0
		
48、下载镜像
[root@k8s-node1 data]# docker pull rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1
v1.0.1: Pulling from rancher/mirrored-flannelcni-flannel-cni-plugin
5758d4e389a3: Pull complete 
5f157e942e60: Pull complete 
Digest: sha256:5dd61f95e28fa7ef897ff2fa402ce283e5078d334401d2f62d00a568f779f2d5
Status: Downloaded newer image for rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1
docker.io/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1
[root@k8s-node1 data]# 
[root@k8s-node1 data]# 
[root@k8s-node1 data]# 
[root@k8s-node1 data]# docker pull rancher/mirrored-flannelcni-flannel:v0.17.0
v0.17.0: Pulling from rancher/mirrored-flannelcni-flannel
5758d4e389a3: Already exists 
052816d6a684: Pull complete 
96c6af7b4b9a: Pull complete 
f28287b73045: Pull complete 
6a87e811090e: Pull complete 
3df5ffe16b5b: Pull complete 
ed7fc7d3dae5: Pull complete 
Digest: sha256:4bf659e449be809763b04f894f53a3d8610e00cf2cd979bb4fffc9470eb40d1b
Status: Downloaded newer image for rancher/mirrored-flannelcni-flannel:v0.17.0
docker.io/rancher/mirrored-flannelcni-flannel:v0.17.0

49、打包传入其他主机，并导入其他主机
[root@k8s-node1 k8s-images]# docker save rancher/mirrored-flannelcni-flannel-cni-plugin:v1.0.1 > flannel-cni-plugin.tar
[root@k8s-node1 k8s-images]# docker save rancher/mirrored-flannelcni-flannel:v0.17.0 > flannel.tar

for i in {130..132};do scp flannel* 192.168.73.$i:/data/data/bao/k8s-images/ ;done

docker load < /data/data/bao/k8s-images/flannel.tar
docker load < /data/data/bao/k8s-images/flannel-cni-plugin.tar

开网
kubectl create -f  kube-flannel.yml
 
 
50、其他在安装过程中的报错
kubectl-proxy报错，在kubeadm-conf.yaml文件中已经更改，若有其他报错，则更改回来，后续报错，则按此更改
server.go:482] failed complete: unrecognized feature gate: SupportIPVSProxyMode
更改k8s的环境
kubectl edit cm kube-proxy -n kube-system
注释掉下面的两行
    #featureGates:
    #  SupportIPVSProxyMode: true
删除kube-proxy pod，等待pod重启
 
 
 
51、创建pod失败报错
network: failed to set bridge addr: "cni0" already has an IP address different from 10.244.3.1/24
cni0网卡和flannel网卡的网段不一样导致的
删除cni0网卡，让其自己重新创建网卡
ifconfig cni0 down    
ip link delete cni0
 
 
52、测试
可以按此简单测试
https://blog.csdn.net/liao__ran/article/details/102647786
https://blog.csdn.net/liao__ran/article/details/106009227


53、去除Master节点污点，使其可以分配Pod资源；
kubectl taint nodes --all node-role.kubernetes.io/master-