安装部署主控节点4层反向代理服务

部署在hdss7-11 hdss7-12机器上，
用VIP 192.168.153.10的7443端口，反代hdss7-21、hdss7-22的apiserver6443端口

HDSS7-11和HDSS7-12上同时操作

nginx配置

[root@hdss7-11 ~]# yum install -y nginx	

[root@hdss7-11 ~]# vi /etc/nginx/nginx.conf		
stream {
    upstream kube-apiserver {
        server 10.4.7.21:6443     max_fails=3 fail_timeout=30s;
        server 10.4.7.22:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

检查配置文件

[root@hdss7-11 ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

[root@hdss7-11 ~]# systemctl start nginx
[root@hdss7-11 ~]# systemctl enable nginx

keepalived安装配置

[root@localhost ~]# yum install -y curl gcc openssl-devel libnl3-devel net-snmp-devel
[root@hdss7-11 ~]# yum install keepalived -y

编写监听脚本 – 直接上传

[root@hdss7-11 ~]# vi /etc/keepalived/check_port.sh	
#!/bin/bash
#keepalived 监控端口脚本
#使用方法：
#在keepalived的配置文件中
#vrrp_script check_port {#创建一个vrrp_script脚本,检查配置
#    script "/etc/keepalived/check_port.sh 6379" #配置监听的端口
#    interval 2 #检查脚本的频率,单位（秒）
#}
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
        PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`
        if [ $PORT_PROCESS -eq 0 ];then
                echo "Port $CHK_PORT Is Not Used,End."
                exit 1
        fi
else
        echo "Check Port Cant Be Empty!"
fi

[root@hdss7-11 ~]# chmod +x /etc/keepalived/check_port.sh

配置keepalived – 删除原文件，直接上传修改

keepalived 主(10.4.7.11):

[root@hdss7-11 ~]# vi /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id 10.4.7.11

}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 10.4.7.11
    nopreempt

    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        10.4.7.10
    }
}

备注： 
interface eth0	# 根据实际网卡更改
10.4.7.10 # 虚拟IP

keepalived从: – 直接上传

[root@hdss7-12 ~]# vi /etc/keepalived/keepalived.conf 

! Configuration File for keepalived
global_defs {
        router_id 10.4.7.12
        script_user root
        enable_script_security
}
vrrp_script chk_nginx {
        script "/etc/keepalived/check_port.sh 7443"
        interval 2
        weight -20
}
vrrp_instance VI_1 {
        state BACKUP
        interface ens33
        virtual_router_id 251
        mcast_src_ip 10.4.7.12
        priority 90
        advert_int 1
        authentication {
                auth_type PASS
                auth_pass 11111111
        }
        track_script {
                chk_nginx
        }
        virtual_ipaddress {
                10.4.7.10
        }
}

[root@hdss7-11 keepalived]# ss -lnt|grep 7443|wc -l         
1

[root@hdss7-11 ~]# systemctl start keepalived
[root@hdss7-11 ~]# systemctl enable keepalived

如果主keepalived服务因故障使得vip飘移到从keepalived，一定要确认主keepalived服务对应的端口和服务都已经恢复，再重启从keepalived服务，这样才能使从vip飘移到主keepalived的vip。
在生产环境中，因vip飘移导致的网络抖动是重大生产事故。

[root@hdss7-11 ~]# netstat -luntp | grep 7443
tcp        0      0 0.0.0.0:7443            0.0.0.0:*               LISTEN      22071/nginx: master 

问题汇总：
1、如果重启keepalived服务没有生效导致无法生成虚拟ip，请检查进程，然后找到进程pid，手动杀死进程再重启即可

[root@hdss7-11 src]# systemctl stop keepalived

[root@hdss7-11 src]# ps -ef  | grep keepalived 
root      13910      1  0 17:59 ?        00:00:00 /usr/sbin/keepalived -D
root      13911  13910  0 17:59 ?        00:00:00 /usr/sbin/keepalived -D
root      13912  13910  0 17:59 ?        00:00:00 /usr/sbin/keepalived -D
root      14354  11683  0 18:01 pts/0    00:00:00 grep --color=auto keepalived

[root@hdss7-11 src]# kill -9 13061

[root@hdss7-11 src]# systemctl start keepalived