环境准备

服务器规划
服务器配置即角色规划如下，操作系统仍然选择 Ubuntu Server X64 18.04
192.168.90.31 4核2G 40G硬盘 Kubernetes server1 Master 主
192.168.90.32 4核2G 40G硬盘 Kubernetes server2 Master 备
192.168.90.33 4核2G 40G硬盘 Kubernetes server3 Master 备
192.168.90.34 4核2G 40G硬盘 Kubernetes server4 Slave
192.168.90.35 4核2G 40G硬盘 Kubernetes server5 Slave
192.168.90.36 4核2G 40G硬盘 Kubernetes server6 Slave

三台master节点通过 vip 192.168.90.100 代理访问

环境准备
按照kubeadm安装K8s集群 中的步骤，安装一台虚拟机并完成初步配置工作，之后再做如下配置：
同步时间
设置时区选择亚洲上海

eric@server1:~$ sudo dpkg-reconfigure tzdata
[sudo] password for eric:

Current default time zone: 'Asia/Shanghai'
Local time is now:      Mon Aug  9 23:05:09 CST 2021.
Universal Time is now:  Mon Aug  9 15:05:09 UTC 2021.

eric@server1:~$ sudo apt-get install ntpdate  --安装 ntpdate
Reading package lists... Done
......
eric@server1:~$ sudo ntpdate cn.pool.ntp.org --设置系统时间与网络时间同步（cn.pool.ntp.org 位于中国的公共 NTP 服务器）
 9 Aug 23:06:30 ntpdate[33117]: adjust time server 202.118.1.130 offset 0.007500 sec
eric@server1:~$ sudo hwclock --systohc  --将系统时间写入硬件时间
eric@server1:~$ date  --查看确认时间
Mon Aug  9 23:06:49 CST 2021

配置IPVS

eric@server1:~$ sudo apt-get install -y ipset ipvsadm  --安装系统工具
Reading package lists... Done
......
eric@server1:~$ sudo mkdir -p /etc/sysconfig/modules/   --创建目录  配置并加载ipvs模块
eric@server1:~$ sudo vi /etc/sysconfig/modules/ipvs.modules --编辑文件并保存
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4

---切换root用户执行脚本否则报错
root@server1:/home/eric# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
ip_vs_sh               16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  0
ip_vs                 151552  6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_defrag_ipv6         20480  1 ip_vs
nf_conntrack_ipv4      16384  4
nf_defrag_ipv4         16384  1 nf_conntrack_ipv4
nf_conntrack          135168  8 xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_ipv4,nf_nat,ipt_MASQUERADE,nf_nat_ipv4,nf_conntrack_netlink,ip_vs
libcrc32c              16384  4 nf_conntrack,nf_nat,raid456,ip_vs4

配置内核参数

root@server1:/home/eric# vi /etc/sysctl.d/k8s.conf  --编辑配置参数
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
root@server1:/home/eric# sysctl --system   ---应用参数
* Applying /etc/sysctl.d/10-console-messages.conf ...
kernel.printk = 4 4 1 7
* Applying /etc/sysctl.d/10-ipv6-privacy.conf ...
net.ipv6.conf.all.use_tempaddr = 2
net.ipv6.conf.default.use_tempaddr = 2
* Applying /etc/sysctl.d/10-kernel-hardening.conf ...
......
* Applying /etc/sysctl.d/k8s.conf ...   --生效
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0

修改 cloud.cfg

vi /etc/cloud/cloud.cfg
# 该配置默认为 false，修改为 true 即可
preserve_hostname: true

克隆虚拟机并分别配置ip和主机名

hostnamectl set-hostname server1  --配置主机名命令

ip配置：找到并修改如下文件，修改保存后 执行 sudo netplan apply 使生效

eric@server1:~$ cat /etc/netplan/00-installer-config.yaml
# This is the network config written by 'subiquity'
network:
  ethernets:
    ens33:
      dhcp4: false
      addresses: [192.168.90.32/24]
      gateway4: 192.168.90.1
      nameservers:
              addresses: [8.8.8.8]
  version: 2

高可用原理

Kubernetes Master 节点运行组件如下：
kube-apiserver： 提供了资源操作的唯一入口，并提供认证、授权、访问控制、API 注册和发现等机制
kube-scheduler： 负责资源的调度，按照预定的调度策略将 Pod 调度到相应的机器上
kube-controller-manager： 负责维护集群的状态，比如故障检测、自动扩展、滚动更新等
etcd： CoreOS 基于 Raft 开发的分布式 key-value 存储，可用于服务发现、共享配置以及一致性保障（如数据库选主、分布式锁等）

kube-scheduler 和 kube-controller-manager 可以以集群模式运行，通过 leader 选举产生一个工作进程，其它进程处于阻塞模式。
kube-apiserver 可以运行多个实例，但对其它组件需要提供统一的访问地址，本章节部署 Kubernetes 高可用集群实际就是利用 HAProxy + Keepalived 配置该组件
配置的思路就是利用 HAProxy + Keepalived 实现 kube-apiserver 虚拟 IP 访问从而实现高可用和负载均衡，拆解如下：
Keepalived 提供 kube-apiserver 对外服务的虚拟 IP（VIP）
HAProxy 监听 Keepalived VIP
运行 Keepalived 和 HAProxy 的节点称为 LB（负载均衡） 节点
Keepalived 是一主多备运行模式，故至少需要两个 LB 节点
Keepalived 在运行过程中周期检查本机的 HAProxy 进程状态，如果检测到 HAProxy 进程异常，则触发重新选主的过程，VIP 将飘移到新选出来的主节点，从而实现 VIP 的高可用
所有组件（如 kubeclt、apiserver、controller-manager、scheduler 等）都通过 VIP +HAProxy 监听的 6444 端口访问 kube-apiserver 服务（注意：kube-apiserver 默认端口为 6443，为了避免冲突我们将 HAProxy 端口设置为 6444，其它组件都是通过该端口统一请求 apiserver）

安装HAProxy和Keepalived

HAproxy启动脚本
master1节点创建HAproxy启动脚本,并设置执行权限

sudo mkdir -p /usr/local/kubernetes/lb
sudo vi /usr/local/kubernetes/lb/start-haproxy.sh

# 输入内容如下
#!/bin/bash
# 修改为你自己的 Master 地址
MasterIP1=192.168.90.31
MasterIP2=192.168.90.32
MasterIP3=192.168.90.33
# 这是 kube-apiserver 默认端口，不用修改
MasterPort=6443

# 容器将 HAProxy 的 6444 端口暴露出去
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
        -e MasterIP1=$MasterIP1 \
        -e MasterIP2=$MasterIP2 \
        -e MasterIP3=$MasterIP3 \
        -e MasterPort=$MasterPort \
        wise2c/haproxy-k8s

# 设置权限
sudo chmod +x /usr/local/kubernetes/lb/start-haproxy.sh

Keepalived启动脚本
master01节点增加 keepalived启动脚本，并添加执行权限如下：

sudo mkdir -p /usr/local/kubernetes/lb
sudo vi /usr/local/kubernetes/lb/start-keepalived.sh
# 输入内容如下
#!/bin/bash
# 修改为你自己的虚拟 IP 地址
VIRTUAL_IP=192.168.90.100
# 虚拟网卡设备名
INTERFACE=ens33
# 虚拟网卡的子网掩码
NETMASK_BIT=24
# HAProxy 暴露端口，内部指向 kube-apiserver 的 6443 端口
CHECK_PORT=6444
# 路由标识符
RID=10
# 虚拟路由标识符
VRID=160
# IPV4 多播地址，默认 224.0.0.18
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
        --net=host --cap-add=NET_ADMIN \
        -e VIRTUAL_IP=$VIRTUAL_IP \
        -e INTERFACE=$INTERFACE \
        -e CHECK_PORT=$CHECK_PORT \
        -e RID=$RID \
        -e VRID=$VRID \
        -e NETMASK_BIT=$NETMASK_BIT \
        -e MCAST_GROUP=$MCAST_GROUP \
        wise2c/keepalived-k8s
# 设置权限
sudo chmod +x /usr/local/kubernetes/lb/start-keepalived.sh

复制脚本到其他两台master
32 和 33 创建 目录 ，并复制脚本文件命令如下

sudo mkdir -p /usr/local/kubernetes/lb
root@server1:/home/eric# scp /usr/local/kubernetes/lb/start-haproxy.sh /usr/local/kubernetes/lb/start-keepalived.sh eric@192.168.90.32:/home/eric   --先复制到服务器 再到服务器上复制到指定目录
root@server1:/home/eric# scp /usr/local/kubernetes/lb/start-haproxy.sh /usr/local/kubernetes/lb/start-keepalived.sh eric@192.168.90.33:/home/eric
eric@server3:~$ sudo mv *.sh /usr/local/kubernetes/lb

启动容器
三个节点分别执行如下命令,docker 会下载、启动 haproxy和keepalived 镜像

sudo sh /usr/local/kubernetes/lb/start-haproxy.sh && sudo  sh /usr/local/kubernetes/lb/start-keepalived.sh

检验容器
三个主节点分别执行 docker ps 可以看到 haproxy和keepalived 正在运行如下：

root@server1:/home/eric# docker ps
CONTAINER ID   IMAGE                   COMMAND                  CREATED              STATUS              PORTS                                       NAMES
2ee95ae52da6   wise2c/keepalived-k8s   "/usr/bin/keepalived…"   52 seconds ago       Up 51 seconds                                                   Keepalived-K8S
97db17bc81c7   wise2c/haproxy-k8s      "/docker-entrypoint.…"   About a minute ago   Up About a minute   0.0.0.0:6444->6444/tcp, :::6444->6444/tcp   HAProxy-K8S

虚拟IP验证
31、32、33 三台服务器 执行如下命令，只有一台可以看到 ip与虚拟ip绑定。如果 被绑定的一台宕机，绑定关系就会漂移到另外两台机器中的一台上，默认在 31 服务器上，关闭 31服务器上会出现在33服务器上如下：

eric@server3:~$ ip a | grep ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    inet 192.168.90.33/24 brd 192.168.90.255 scope global ens33
    inet 192.168.90.100/24 scope global secondary ens33

部署K8S集群

创建工作目录并导出配置文件

# 创建工作目录
sudo  mkdir -p /usr/local/kubernetes/cluster
# 导出配置文件到工作目录
su root
 kubeadm config print init-defaults --kubeconfig ClusterConfiguration > /usr/local/kubernetes/cluster/kubeadm.yml

修改配置文件
33节点修改kubeadm.yml 内容如下

root@server1:/usr/local/kubernetes/cluster# cat kubeadm.yml
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.90.33  #节点ip
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: server1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta1
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.90.100:6444"  # vip 和 端口
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers  # 阿里镜像库
kind: ClusterConfiguration
kubernetesVersion: v1.14.10   # 版本号
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"  # IP段 不能和 主节点所在ip段冲突  如：主节点ip 为 192.168.90.33 那么这里不能谢 192.168.0.0/16  
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
# 开启 IPVS 模式
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

kubeadm 初始化master

kubeadm 初始化

root@server1:/usr/local/kubernetes/cluster# kubeadm init --config=kubeadm.yml --experimental-upload-certs | tee kubeadm-init.log
......
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.90.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:d5890a0d44846cb7b18ae919a04031c5290d002769a93892a79bb427f657fe9e \
    --experimental-control-plane --certificate-key cf231517325f3c8756e057c8851d2065363a875cccea31c5629871a44c394dbf

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --experimental-upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.90.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:d5890a0d44846cb7b18ae919a04031c5290d002769a93892a79bb427f657fe9e

根据日志输出,切换到普通用户eric执行以下命令

root@server3:/usr/local/kubernetes/cluster# su eric
eric@server3:/usr/local/kubernetes/cluster$ mkdir -p $HOME/.kube
eric@server3:/usr/local/kubernetes/cluster$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
eric@server3:/usr/local/kubernetes/cluster$   sudo chown $(id -u):$(id -g) $HOME/.kube/config

验证是否成功

eric@server3:/usr/local/kubernetes/cluster$ kubectl get node
NAME      STATUS     ROLES    AGE     VERSION
server3   NotReady   master   4m11s   v1.14.10

安装网络插件

eric@server3:/usr/local/kubernetes/cluster$ kubectl apply -f https://docs.projectcalico.org/v3.7/manifests/calico.yaml
configmap/calico-config created
......
serviceaccount/calico-node created
deployment.extensions/calico-kube-controllers created
serviceaccount/calico-kube-controllers created

# 验证安装是否成功  我这里足足等了 64分钟 各个插件才正常运行 running 状态
watch kubectl get pods --all-namespaces
kube-system   kube-scheduler-server3                    1/1     Running             0          34m
Every 2.0s: kubectl get pods --all-namespaces                                                                                                                                                                                                           server3: Sun Aug 15 00:59:23 2021
NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-f6ff9cbbb-6dcjs   1/1     Running   0          64m
kube-system   calico-node-sb2kb                         1/1     Running   0          64m
kube-system   coredns-7b7df549dd-vmpww                  1/1     Running   0          66m
kube-system   coredns-7b7df549dd-zzjf8                  1/1     Running   0          66m
kube-system   etcd-server3                              1/1     Running   0          65m
kube-system   kube-apiserver-server3                    1/1     Running   0          65m
kube-system   kube-controller-manager-server3           1/1     Running   0          65m
kube-system   kube-proxy-q42pg                          1/1     Running   0          66m
kube-system   kube-scheduler-server3                    1/1     Running   0          65m

加入mater节点
31 和32节点分别执行初始化日志中的 主节点加入命令，将 31 和 32 节点初始化成 master节点。
注意：如果初始化完成很久之后才执行 加入master节点操作，那么token 可能会失效，参考上一篇文章，重新获取token 等参数

 kubeadm join 192.168.90.100:6444 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:d5890a0d44846cb7b18ae919a04031c5290d002769a93892a79bb427f657fe9e \
    --experimental-control-plane --certificate-key cf231517325f3c8756e057c8851d2065363a875cccea31c5629871a44c394dbf

.....
[mark-control-plane] Marking the node server1 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node server1 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.

To start administering your cluster from this node, you need to run the following as a regular user:

	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.

按照上述日志，执行配置命令：

root@server1:/home/eric# su eric
eric@server1:~$ mkdir -p $HOME/.kube
eric@server1:~$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[sudo] password for eric:
eric@server1:~$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

加入node节点
三个从节点分别执行以下命令，加入集群
初始化日志中会打印加入命令，直接复制执行即可，如果参数不正确，参考上一篇，重新生成参数。

root@server4:/home/eric# kubeadm join 192.168.90.100:6444 --token abcdef.0123456789abcdef \
>     --discovery-token-ca-cert-hash sha256:19c012298212324b7851d89d71af9ff0d50c4fb130cb774b8a80c3a32d51d051
[preflight] Running pre-flight checks
	[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
	[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.8. Latest validated version: 18.09
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.14" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Activating the kubelet service
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

验证集群状态

master节点执行如下命令验证集群状态

eric@server1:~$ kubectl get nodes   --查看节点
NAME      STATUS     ROLES    AGE     VERSION
server1   Ready      master   7m35s   v1.14.10
server2   Ready      master   7m22s   v1.14.10
server3   Ready      master   85m     v1.14.10
server4   NotReady   <none>   43s     v1.14.10
server5   NotReady   <none>   42s     v1.14.10
server6   NotReady   <none>   41s     v1.14.10
eric@server1:~$ kubectl get nodes -o wide  --查看节点
NAME      STATUS     ROLES    AGE     VERSION    INTERNAL-IP     EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
server1   Ready      master   9m43s   v1.14.10   192.168.90.31   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server2   Ready      master   9m30s   v1.14.10   192.168.90.32   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server3   Ready      master   87m     v1.14.10   192.168.90.33   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server4   NotReady   <none>   2m51s   v1.14.10   192.168.90.34   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server5   NotReady   <none>   2m50s   v1.14.10   192.168.90.35   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server6   NotReady   <none>   2m49s   v1.14.10   192.168.90.36   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
eric@server1:~$ kubectl -n kube-system get pod -o wide    --查看pod
NAME                                      READY   STATUS                  RESTARTS   AGE     IP                NODE      NOMINATED NODE   READINESS GATES
calico-kube-controllers-f6ff9cbbb-6dcjs   1/1     Running                 0          86m     192.168.141.193   server3   <none>           <none>
calico-node-49lqn                         0/1     PodInitializing         0          10m     192.168.90.31     server1   <none>           <none>
calico-node-jmp28                         0/1     Init:ImagePullBackOff   0          3m17s   192.168.90.36     server6   <none>           <none>
calico-node-kszl7                         0/1     Init:0/2                0          3m18s   192.168.90.35     server5   <none>           <none>
calico-node-njz8v                         0/1     PodInitializing         0          9m58s   192.168.90.32     server2   <none>           <none>
calico-node-sb2kb                         1/1     Running                 0          86m     192.168.90.33     server3   <none>           <none>
calico-node-sn874                         0/1     Init:0/2                0          3m19s   192.168.90.34     server4   <none>           <none>
coredns-7b7df549dd-vmpww                  1/1     Running                 0          87m     192.168.141.194   server3   <none>           <none>
coredns-7b7df549dd-zzjf8                  1/1     Running                 0          87m     192.168.141.195   server3   <none>           <none>
etcd-server1                              1/1     Running                 0          10m     192.168.90.31     server1   <none>           <none>
etcd-server2                              1/1     Running                 0          9m57s   192.168.90.32     server2   <none>           <none>
etcd-server3                              1/1     Running                 0          86m     192.168.90.33     server3   <none>           <none>
kube-apiserver-server1                    1/1     Running                 0          10m     192.168.90.31     server1   <none>           <none>
kube-apiserver-server2                    1/1     Running                 0          9m58s   192.168.90.32     server2   <none>           <none>
kube-apiserver-server3                    1/1     Running                 0          86m     192.168.90.33     server3   <none>           <none>
kube-controller-manager-server1           1/1     Running                 0          10m     192.168.90.31     server1   <none>           <none>
kube-controller-manager-server2           1/1     Running                 0          9m57s   192.168.90.32     server2   <none>           <none>
kube-controller-manager-server3           1/1     Running                 0          86m     192.168.90.33     server3   <none>           <none>
kube-proxy-5hl76                          1/1     Running                 0          10m     192.168.90.31     server1   <none>           <none>
kube-proxy-gt6bj                          1/1     Running                 0          3m19s   192.168.90.34     server4   <none>           <none>
kube-proxy-nxx9l                          1/1     Running                 0          3m17s   192.168.90.36     server6   <none>           <none>
kube-proxy-q42pg                          1/1     Running                 0          87m     192.168.90.33     server3   <none>           <none>
kube-proxy-qfkth                          1/1     Running                 0          9m58s   192.168.90.32     server2   <none>           <none>
kube-proxy-zc5c2                          1/1     Running                 0          3m18s   192.168.90.35     server5   <none>           <none>
kube-scheduler-server1                    1/1     Running                 0          10m     192.168.90.31     server1   <none>           <none>
kube-scheduler-server2                    1/1     Running                 0          9m58s   192.168.90.32     server2   <none>           <none>
kube-scheduler-server3                    1/1     Running                 0          87m     192.168.90.33     server3   <none>           <none>
eric@server1:~$ kubectl -n kube-system get svc  --查看服务
NAME       TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
kube-dns   ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP,9153/TCP   88m

eric@server1:~$ kubectl -n kube-system exec etcd-kubernetes-master-01 -- etcdctl \    --查看etcd集群状态
> --endpoints=https://192.168.141.150:2379 \
> --ca-file=/etc/kubernetes/pki/etcd/ca.crt \
> --cert-file=/etc/kubernetes/pki/etcd/server.crt \
> --key-file=/etc/kubernetes/pki/etcd/server.key cluster-health
Error from server (NotFound): pods "etcd-kubernetes-master-01" not found
eric@server1:~$ kubectl -n kube-system exec etcd-server1  -- etcdctl --endpoints=https://192.168.90.31:2379 --ca-file=/etc/kubernetes/pki/etcd/ca.crt --cert-file=/etc/kubernetes/pki/etcd/server.crt --key-file=/etc/kubernetes/pki/etcd/server.key cluster-health
member 5054125c1f93982 is healthy: got healthy result from https://192.168.90.33:2379
member 35577abe54c175af is healthy: got healthy result from https://192.168.90.32:2379
member 6f5d23fdfa6c99f4 is healthy: got healthy result from https://192.168.90.31:2379
cluster is healthy

验证高可用

Keepalived 要求至少 2 个备用节点，故想测试高可用至少需要 1 主 2 从模式验证，否则可能出现意想不到的问题
开始 通过ip a |grep ens 命令可以看到 vip 在 33节点上，即 33节点作为master 对外提供服务，在 31 和 32 节点上 可以通过 kubectl get nodes -o wide 查询到节点信息 ，证明调用了主节点的apiserver服务。通过在33节点shutdown模拟现行主节点宕机，

shutdown -h now  --关机

通过ip a |grep ens 命令可以看到 vip 漂移到了 32 节点

eric@server2:~$ ip a|grep ens
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    inet 192.168.90.32/24 brd 192.168.90.255 scope global ens33
    inet 192.168.90.100/24 scope global secondary ens33

这时在 31节点仍然可以通过 kubectl get nodes -o wide 获取到节点信息如下，证明33节点宕机情况下，api server 服务仍然可用：

eric@server1:~$ kubectl get nodes -o wide
NAME      STATUS     ROLES    AGE    VERSION    INTERNAL-IP     EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
server1   Ready      master   42m    v1.14.10   192.168.90.31   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server2   Ready      master   42m    v1.14.10   192.168.90.32   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server3   NotReady   master   120m   v1.14.10   192.168.90.33   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server4   Ready      <none>   35m    v1.14.10   192.168.90.34   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server5   Ready      <none>   35m    v1.14.10   192.168.90.35   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8
server6   Ready      <none>   35m    v1.14.10   192.168.90.36   <none>        Ubuntu 18.04.5 LTS   4.15.0-153-generic   docker://20.10.8

配置运行nginx容器

部署deployment
创建 配置文件nginx-deployment.yaml如下：

eric@server1:/usr/local/kubernetes/cluster$ cat nginx-deployment.yml
# API 版本号
apiVersion: extensions/v1beta1
# 类型，如：Pod/ReplicationController/Deployment/Service/Ingress
kind: Deployment
# 元数据
metadata:
  # Kind 的名称
  name: nginx-app
spec:
  # 部署的实例数量
  replicas: 2
  template:
    metadata:
      labels:
        # 容器标签的名字，发布 Service 时，selector 需要和这里对应
        name: nginx
    spec:
      # 配置容器，数组类型，说明可以配置多个容器
      containers:
      # 容器名称
      - name: nginx
        # 容器镜像
        image: nginx
        # 暴露端口
        ports:
        # Pod 端口
        - containerPort: 80

添加部署

eric@server3:/usr/local/kubernetes/cluster$ kubectl create -f nginx-deployment.yml   
deployment.extensions/nginx-app created

删除部署命令

kubectl delete -f nginx-deployment.yml

发布service
nginx-service.yml配置文件如下：

# API 版本号
apiVersion: v1
# 类型，如：Pod/ReplicationController/Deployment/Service/Ingress
kind: Service
# 元数据
metadata:
  # Kind 的名称
  name: nginx-http
spec:
  # 暴露端口
  ports:
    ## Service 暴露的端口
    - port: 80
      ## Pod 上的端口，这里是将 Service 暴露的端口转发到 Pod 端口上
      targetPort: 80
  # 类型
  type: LoadBalancer
  # 标签选择器
  selector:
    # 需要和上面部署的 Deployment 标签名对应
    name: nginx

eric@server3:/usr/local/kubernetes/cluster$ kubectl create -f  nginx-service.yml

也可以deployment 和service一起部署
配置文件合并在一起 内容使用 — 分割即可

查看验证

eric@server3:/usr/local/kubernetes/cluster$ kubectl get pods
NAME                         READY   STATUS    RESTARTS   AGE
nginx-app-64bb598779-kfqm2   1/1     Running   0          4m10s
nginx-app-64bb598779-qzsjp   1/1     Running   0          4m10s
eric@server3:/usr/local/kubernetes/cluster$ kubectl get deployment
NAME        READY   UP-TO-DATE   AVAILABLE   AGE
nginx-app   2/2     2            2           4m27s
eric@server3:/usr/local/kubernetes/cluster$ kubectl get service
NAME         TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP      10.96.0.1      <none>        443/TCP        11h
nginx-http   LoadBalancer   10.99.153.51   <pending>     80:31467/TCP   47s
eric@server3:/usr/local/kubernetes/cluster$ kubectl describe service nginx-http
Name:                     nginx-http
Namespace:                default
Labels:                   <none>
Annotations:              <none>
Selector:                 name=nginx
Type:                     LoadBalancer
IP:                       10.99.153.51
Port:                     <unset>  80/TCP
TargetPort:               80/TCP
NodePort:                 <unset>  31467/TCP
Endpoints:                192.168.205.67:80,192.168.22.3:80
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>

访问192.168.90.31/32/33:31467 可以访问到nginx页面