1. K8S高可用集群概念

K8s apierver提供了k8s各类资源对象（pod,RC,Service等）的增删改查及watch等HTTP Rest接口，是整个系统的数据总线和数据中心。
Apiserver是用户和 kubernetes 集群交互的入口，封装了核心对象的增删改查操作，提供了 RESTFul 风格的 API 接口，通过etcd来实现持久化并维护对象的一致性。所以在整个K8S集群中，Apiserver服务至关重要，一旦宕机，整个K8S平台将无法使用，所以保障企业高可用是运维必备的工作之一。

1.1 K8S高可用工作原理

我们将运行 keepalived 和 haproxy 的节点称为 LB（Load balancer） 节点。由于 keepalived 是一主多备运行模式，故至少两个 LB 节点。此处采用Master节点的2台机器，Haproxy 监听的端口(8443) 需要与 kube-apiserver 的端口 6443 不同，避免冲突。
基于Keepalived 和Haproxy 实现 Kube-apiserver 高可用的步骤：

1. Keepalived 提供 kube-apiserver 对外服务的 VIP；

2. Haproxy 监听 VIP，后端连接所有 kube-apiserver 实例，提供健康检查和负载均衡功能；

   Keepalived 在运行过程中周期检查本机的 haproxy 进程状态，如果检测到 haproxy 进程异常，则触发重新选主的过程，VIP 将飘移到新选出来的主节点，从而实现 VIP 的高可用。
   最终实现所有组件（如 kubectl、kube-controller-manager、kube-scheduler、kube-proxy 等）都通过 VIP+Haproxy 监听的 8443 端口访问 kube-apiserver服务。

2.部署keepalived+Haproxy

2.1 实验环境

在已经安装好的k8s集群中进行高可用的实现
还可以在后面加入更多的node节点 因为资源问题 只能这样演示

角色	IP	安装步骤
master1	192.168.1.81	1.部署haproxy 2.部署 keepalived 3.kubeadm初始化操作
master2	192.168.1.82	1.部署haproxy 2.部署 keepalived 3.kubeadm初始化操作
master3	192.168.1.83	1.部署haproxy 2.部署 keepalived 3.kubeadm初始化操作
vip	192.168.1.188

master1 2 3 互为k8s集群内的成员

2.1 修改k8s主机名

在三台机器都执行

#添加hosts解析；
cat >/etc/hosts<<EOF
127.0.0.1 localhost localhost.localdomain
192.168.1.81 master1
192.168.1.82 master2
192.168.1.83 master3
EOF

[root@master1 ~]# hostnamectl set-hostname master1

[root@node1 ~]# hostnamectl set-hostname master2
[root@node1 ~]# bash

[root@node2 ~]# hostnamectl set-hostname master3
[root@node2 ~]# bash

2.2 安装Haproxy

2.2.1 yum安装

[root@master1 ~]#  yum install haproxy

2.2.2 修改配置文件

[root@master1 ~]# cd /etc/haproxy/
[root@master1 haproxy]# ls
haproxy.cfg
[root@master1 haproxy]# vim haproxy.cfg 
global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /etc/haproxy
    stats socket /var/lib/haproxy/haproxy-admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    nbproc 1
defaults
    log     global
    timeout connect 5000
    timeout client  10m
    timeout server  10m
listen  admin_stats
    bind 0.0.0.0:10080
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login\ Haproxy
    stats auth admin:123456
    stats hide-version
    stats admin if TRUE
listen kube-master
    bind 0.0.0.0:8443
    mode tcp
    option tcplog
    balance source
    server master1 192.168.1.81:6443 check inter 2000 fall 2 rise 2 weight 1
    server master2 192.168.1.82:6443 check inter 2000 fall 2 rise 2 weight 1
    server master3 192.168.1.83:6443 check inter 2000 fall 2 rise 2 weight 1

将配置文件传送到master2 3

[root@master1 haproxy]# scp haproxy.cfg 192.168.1.83:/etc/haproxy/
root@192.168.1.83's password: 
haproxy.cfg              
[root@master1 haproxy]# scp haproxy.cfg 192.168.1.82:/etc/haproxy/
root@192.168.1.82's password: 
haproxy.cfg                                                                                        100%  917   191.9KB/s   00:00    
[root@master1 haproxy]# 

[root@master1 ~]# ps -ef |grep haproxy
root     112052      1  0 17:26 ?        00:00:00 /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid
haproxy  112053 112052  0 17:26 ?        00:00:00 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds
haproxy  112054 112053  0 17:26 ?        00:00:00 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid -Ds
root     112832 101523  0 17:28 pts/0    00:00:00 grep --color=auto haproxy
[root@master1 ~]# 

2.3 安装keepalived、

2.3.1 yum安装

 yum install keepalived.x86_64 -y

2.3.2 配置文件修改

在master配置优先级为100 在backup为90

! Configuration File for keepalived 
 global_defs { 
 notification_email { 
      zflgood@139.com 
 } 
    notification_email_from zflgood@139.com 
    smtp_server 127.0.0.1 
    smtp_connect_timeout 30 
    router_id LVS_DEVEL 
 } 
 vrrp_script chk_haproxy { 
    script "/data/sh/check_haproxy.sh" 
   interval 2 
   weight 2 
 } 
 # VIP1 
 vrrp_instance VI_1 { 
     state  MASTER
     interface ens33 
     virtual_router_id 151 
     priority 100 
     advert_int 5 
     nopreempt 
     authentication { 
         auth_type  PASS 
         auth_pass  2222 
     } 
     virtual_ipaddress { 
         192.168.1.188 
     } 
     track_script { 
     chk_haproxy 
    } 
 }

2.3.3 创建chk_haproxy 脚本

#!/bin/bash
CHECK_NUM=$(ps -ef |grep -aiE haproxy|grep -vE "grep|check"|wc -l )
if [ $CHECK_NUM -eq 0 ];then
        systemctl stop keepalived.service
fi

 chmod +x /data/sh/check_haproxy.sh 

将脚本传送到master2 和master3

[root@master1 ~]# scp -r /data/sh/check_haproxy.sh 192.168.1.82:
/data/sh/check_haproxy.sh
root@192.168.1.82's password: 
check_haproxy.sh              100%  155    45.8KB/s   00:00    
[root@master1 ~]# scp -r /data/sh/check_haproxy.sh 192.168.1.83:/data/sh/check_haproxy.sh
root@192.168.1.83's password: 
check_haproxy.sh              100%  155    52.3KB/s   00:00    
[root@master1 ~]# 

2.3.4 启动keepalived

master1出现

[root@master1 keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:0d:06:5b brd ff:ff:ff:ff:ff:ff
    inet 192.168.1.81/24 brd 192.168.1.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.1.188/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe0d:65b/64 scope link 
       valid_lft forever preferred_lft forever

2.3.5 检查keepalived和haproxy的高可用

kill掉master1的haproxy

[root@master1 keepalived]# pkill haproxy
[root@master1 keepalived]# ps -ef |grep !$
ps -ef |grep haproxy
root     122014 121461  0 17:50 pts/0    00:00:00 grep --color=auto haproxy
[root@master1 keepalived]# 

master2 立马成为了新的vip节点

Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.1.188
Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:26 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188
Apr 15 18:00:31 node1 Keepalived_vrrp[55309]: Sending gratuitous ARP on ens33 for 192.168.1.188

3. 初始化Master集群

3.1 Apiserver端口统一修改为新VIP+8443

首先要重置集群

kubeadm reset

在其中一台机器执行

#初始化集群，同时使用VIP+8443端口；
kubeadm init   --control-plane-endpoint=192.168.1.188:8443 --image-repository registry.aliyuncs.com/google_containers   --kubernetes-version v1.20.4   --service-cidr=10.10.0.0/16   --pod-network-cidr=10.244.0.0/16   --upload-certs

3.2 将新的master加入到集群

重点一定是在其中一台机器进行初始化
其他两台加入进来就可以

rm -rf $HOME/.kube

kubeadm join 192.168.1.188:8443 --token e2ikp7.02wu2we99c5bdmgs \
>     --discovery-token-ca-cert-hash sha256:59ce337118a63a8b4c94d3eb1689f0c46c76a67cf864decb1d8fea11cc59000d \
>     --control-plane --certificate-key ac204df97642cc0af7d273e5275d9e77fac227f3f1062eaa66fbccde6cac9757

查看节点 三台masters

[root@master1 ~]# kubectl get nodes
NAME      STATUS   ROLES                  AGE     VERSION
master1   Ready    control-plane,master   5m46s   v1.20.4
master2   Ready    control-plane,master   63s     v1.20.4
master3   Ready    control-plane,master   60s     v1.20.4
[root@master1 ~]# 

[root@master2 ~]# kubectl get nodes
NAME      STATUS   ROLES                  AGE   VERSION
master1   Ready    control-plane,master   25m   v1.20.4
master2   Ready    control-plane,master   20m   v1.20.4
master3   Ready    control-plane,master   70s   v1.20.4
[root@master2 ~]# 

[root@master3 ~]# kubectl get nodes
NAME      STATUS   ROLES                  AGE   VERSION
master1   Ready    control-plane,master   25m   v1.20.4
master2   Ready    control-plane,master   20m   v1.20.4
master3   Ready    control-plane,master   62s   v1.20.4

3.3 检测

前面知道我们的vip在master1 所以在master1 安装pod在kill掉master1 看集群是否还在

3.3.1 创建一个pod

[root@master1 ~]# vim nginx.yaml 

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx
  name: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx
        name: nginx

[root@master1 ~]# kubectl apply -f nginx.yaml 
deployment.apps/nginx created

[root@master1 ~]# kubectl get pods
NAME                     READY   STATUS    RESTARTS   AGE
nginx-6799fc88d8-jq5zl   0/1     Pending   0          28s

一直为pending
因为集群都为master 都带有标记 不在master创建pod的标记 因此需要去除标记

当前的master节点被打上了node-role.kubernetes.io/master:NoSchedule的污点：
[root@master1 ~]#  kubectl describe nodes master1 |grep -E '(Roles|Taints)'
Roles:              control-plane,master
Taints:             node-role.kubernetes.io/master:NoSchedule
[root@master1 ~]# 

执行去除污点的标记

kubectl taint nodes master1 node-role.kubernetes.io/master-

[root@master1 ~]# kubectl taint nodes master1 node-role.kubernetes.io/master-
node/master1 untainted
[root@master1 ~]#  kubectl describe nodes master1 |grep -E '(Roles|Taints)'
Roles:              control-plane,master
Taints:             <none>

[root@master1 ~]# kubectl get pods -o wide
NAME                     READY   STATUS    RESTARTS   AGE     IP           NODE      NOMINATED NODE   READINESS GATES
nginx-6799fc88d8-jq5zl   1/1     Running   0          8m51s   10.244.0.8   master1   <none>           <none>
[root@master1 ~]# kubectl get pods 
NAME                     READY   STATUS    RESTARTS   AGE
nginx-6799fc88d8-jq5zl   1/1     Running   0          9m28s
[root@master1 ~]# 

3.3.2 关闭master1主机

[root@master1 ~]# poweroff
Connection closing...Socket close.

Connection closed by foreign host.

Disconnected from remote host(192.168.1.81) at 19:00:22.

Type `help' to learn how to use Xshell prompt.
[C:\~]$ 

[root@master2 ~]# kubectl get nodes
NAME      STATUS     ROLES                  AGE   VERSION
master1   NotReady   control-plane,master   41m   v1.20.4
master2   Ready      control-plane,master   36m   v1.20.4
master3   Ready      control-plane,master   17m   v1.20.4
[root@master2 ~]# 

其他master依旧可以操作k8s集群 不会在因为主master的问题 而出现问题