多节点部署

//先具备单master节点部署环境
接K8S单节点部署

master01:192.168.20.10 kube-apiserver kube-controller-manager kube-scheduler etcd
master02:192.168.20.40 kube-apiserver kube-controller-manager kube-scheduler
node1:192.168.20.20 kubelet kube-proxy docker flannel etcd
node2:192.168.20.30 kubelet kube-proxy docker flannel etcd

master02部署

[root@localhost ~]# hostnamectl set-hostname master2
[root@localhost ~]# su 

//优先关闭防火墙和selinux服务

[root@master2 ~]# iptables -F
[root@master2 ~]# setenforce 0

//在master01上操作
//复制kubernetes目录到master02

[root@master ~]# scp -r /opt/kubernetes/ root@192.168.20.40:/opt
The authenticity of host '192.168.20.40 (192.168.20.40)' can't be established.
ECDSA key fingerprint is SHA256:v7t4p3JJLUnXziTqE64SOtmKTkJdbSB2hEykd+xG22c.
ECDSA key fingerprint is MD5:85:90:0a:05:38:e9:e3:37:25:de:f0:08:71:9e:9d:c5.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.20.40' (ECDSA) to the list of known hosts.
root@192.168.20.40's password: 
token.csv                                              100%   84    14.3KB/s   00:00    
kube-apiserver                                         100%  929    96.4KB/s   00:00    
kube-scheduler                                         100%   94    21.3KB/s   00:00    
kube-controller-manager                                100%  483    82.4KB/s   00:00    
kube-apiserver                                                                                     100%  184MB  20.4MB/s   00:09    
kubectl                                                                                            100%   55MB  13.7MB/s   00:04    
kube-controller-manager                                                                            100%  155MB  16.9MB/s   00:09    
kube-scheduler                                                                                     100%   55MB  18.6MB/s   00:02    
ca-key.pem                                                                                         100% 1679   341.9KB/s   00:00    
ca.pem                                                                                             100% 1359   192.7KB/s   00:00    
server-key.pem                                                                                     100% 1679   301.5KB/s   00:00    
server.pem                                                                                         100% 1643   224.1KB/s   00:00    

//复制master中的三个组件启动脚本kube-apiserver.service kube-controller-

manager.service	kube-scheduler.service  
[root@master ~]# scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.20.40:/usr/lib/systemd/system/
root@192.168.20.40's password: 
kube-apiserver.service                                                                             100%  282    72.7KB/s   00:00    
kube-controller-manager.service                                                                    100%  317    55.7KB/s   00:00    
kube-scheduler.service                                                                             100%  281    42.4KB/s   00:00    

//master02上操作
//修改配置文件kube-apiserver中的IP

[root@master2 ~]# cd /opt/kubernetes/cfg/
[root@master2 cfg]# vim kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.20.10:2379,https://192.168.20.20:2379,https://192.168.20.30:2379 \
--bind-address=192.168.20.40 \		//这里改成master2的地址
--secure-port=6443 \
--advertise-address=192.168.20.40 \		//这里改成master2的地址
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem"

//特别注意：master02一定要有etcd证书
//需要拷贝master01上已有的etcd证书给master02使用
//在master01上操作

[root@master ~]# scp -r /opt/etcd/ root@192.168.20.40:/opt/
root@192.168.20.40's password: 
etcd                                                                                               100%  509    57.4KB/s   00:00    
etcd                                                                                               100%   18MB  19.9MB/s   00:00    
etcdctl                                                                                            100%   15MB  15.1MB/s   00:01    
ca-key.pem                                                                                         100% 1675   144.8KB/s   00:00    
ca.pem                                                                                             100% 1265   371.3KB/s   00:00    
server-key.pem                                                                                     100% 1679   287.1KB/s   00:00    
server.pem                                                                                         100% 1338   439.4KB/s   00:00 
      

//在master02上操作
//启动master02中的三个组件服务

[root@master2 cfg]# systemctl start kube-apiserver.service 
[root@master2 cfg]# systemctl start kube-controller-manager.service 
[root@master2 cfg]# systemctl start kube-scheduler.service 

//增加环境变量

[root@master2 cfg]# vim /etc/profile
#末尾添加
export PATH=$PATH:/opt/kubernetes/bin/

[root@master2 cfg]# source /etc/profile
[root@master2 cfg]# kubectl get node
NAME            STATUS   ROLES    AGE     VERSION
192.168.20.20   Ready    <none>   7d22h   v1.12.3
192.168.20.30   Ready    <none>   7d21h   v1.12.3

负载均衡部署

//先具备多master节点部署环境
接K8S多节点部署
master1:192.168.20.10 kube-apiserver kube-controller-manager kube-scheduler etcd
master2:192.168.20.40
node1:192.168.20.20 kubelet kube-proxy docker flannel etcd
node2:192.168.20.30 kubelet kube-proxy docker flannel etcd
漂移地址:192.168.20.111
负载均衡LoadBalance
lb01:master 192.168.20.50
lb02:backup 192.168.20.60

//lb01 lb02操作
192.168.20.50

[root@localhost ~]# hostnamectl set-hostname lb01
[root@localhost ~]# su

192.168.20.60

[root@localhost ~]# hostnamectl set-hostname lb02
[root@localhost ~]# su

//优先关闭lb01和lb02的防火墙和selinux服务

systemctl stop firewalld.service 
setenforce 0

//安装nginx服务，把nginx.sh和keepalived.conf脚本拷贝到家目录

vim /etc/yum.repos.d/nginx.repo

[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0

yum install nginx -y

//添加四层转发（在events {}http {}中间添加）
vim /etc/nginx/nginx.conf 

events {
    worker_connections  1024;
}

stream {

   log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log  /var/log/nginx/k8s-access.log  main;

    upstream k8s-apiserver {
        server 192.168.20.10:6443;	//master01地址
        server 192.168.20.40:6443;	//master02地址
    }
    server {
                listen 6443;
                proxy_pass k8s-apiserver;
    }
    }
	
http {

//启动nginx服务

systemctl start nginx

//部署keepalived服务

yum install keepalived -y

//修改配置文件

cp keepalived.conf /etc/keepalived/keepalived.conf 
cp：是否覆盖"/etc/keepalived/keepalived.conf"？ yes

//注意：lb01是Mster配置如下：

[root@lb01 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived 
 
global_defs { 
   # 接收邮件地址 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   # 邮件发送地址 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
    script "/etc/nginx/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.20.111/24 
    } 
    track_script {
        check_nginx
    } 
}

//注意：lb02是Backup配置如下：

[root@lb02 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived 
 
global_defs { 
   # 接收邮件地址 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   # 邮件发送地址 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
    script "/etc/nginx/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.20.111/24 
    } 
    track_script {
        check_nginx
    } 
}

vim /etc/nginx/check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    systemctl stop keepalived
fi

chmod +x /etc/nginx/check_nginx.sh
systemctl start keepalived

//查看lb01地址信息

[root@lb01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:7f:10:c4 brd ff:ff:ff:ff:ff:ff
    inet 192.168.20.50/24 brd 192.168.20.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.20.111/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::2699:92a4:f2b8:7f88/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
    link/ether 52:54:00:64:d5:ef brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000
    link/ether 52:54:00:64:d5:ef brd ff:ff:ff:ff:ff:ff

//漂移地址在lb01中

//查看lb02地址信息

[root@lb02 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:6e:8e:cd brd ff:ff:ff:ff:ff:ff
    inet 192.168.20.60/24 brd 192.168.20.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::761e:cabc:c27a:4cd4/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
    link/ether 52:54:00:c1:04:69 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000
    link/ether 52:54:00:c1:04:69 brd ff:ff:ff:ff:ff:ff

//验证地址漂移（lb01中使用pkill nginx，再在lb02中使用ip a 查看）
//恢复操作（在lb01中先启动nginx服务，再启动keepalived服务）
//nginx站点/usr/share/nginx/html

//在node01和node02上操作
//开始修改node节点配置文件统一VIP（bootstrap.kubeconfig,kubelet.kubeconfig）

vim /opt/kubernetes/cfg/bootstrap.kubeconfig
vim /opt/kubernetes/cfg/kubelet.kubeconfig
vim /opt/kubernetes/cfg/kube-proxy.kubeconfig

//统统修改为VIP

server: https://192.168.20.111:6443

systemctl restart kubelet.service 
systemctl restart kube-proxy.service 

//替换完成直接自检

cd /opt/kubernetes/cfg/
grep 111 *

[root@node1 ~]# cd /opt/kubernetes/cfg/
[root@node1 cfg]# grep 111 *
bootstrap.kubeconfig:    server: https://192.168.20.111:6443
kubelet.kubeconfig:    server: https://192.168.20.111:6443
kube-proxy.kubeconfig:    server: https://192.168.20.111:6443

[root@node2 ~]# cd /opt/kubernetes/cfg/
[root@node2 cfg]# grep 111 *
bootstrap.kubeconfig:    server: https://192.168.20.111:6443
kubelet.kubeconfig:    server: https://192.168.20.111:6443
kube-proxy.kubeconfig:    server: https://192.168.20.111:6443

//在lb01上
//查看nginx的k8s日志

[root@lb01 ~]# tail /var/log/nginx/k8s-access.log 
192.168.20.20 192.168.20.40:6443 - [08/Oct/2020:00:23:11 +0800] 200 1119
192.168.20.20 192.168.20.40:6443 - [08/Oct/2020:00:23:11 +0800] 200 1118
192.168.20.30 192.168.20.40:6443 - [08/Oct/2020:00:23:16 +0800] 200 1120
192.168.20.30 192.168.20.40:6443 - [08/Oct/2020:00:23:16 +0800] 200 1120

//在master01上操作
//测试创建pod

kubectl run nginx --image=nginx
kubectl run --generator=deployment/apps.v1beta1 is DEPRECATED and will be removed in a future version. Use kubectl create instead.
deployment.apps/nginx created

//查看状态

[root@master ~]# kubectl get pods
NAME                    READY   STATUS              RESTARTS   AGE
nginx-dbddb74b8-s4rdt   0/1     ContainerCreating   0          33s   //正在创建中

[root@master2 cfg]# kubectl get pods
NAME                    READY   STATUS    RESTARTS   AGE
nginx-dbddb74b8-s4rdt   1/1     Running   0          80s  //创建完成，运行中

//注意日志问题

[root@master ~]# kubectl logs nginx-dbddb74b8-s4rdt
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-s4rdt)
[root@master ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
clusterrolebinding.rbac.authorization.k8s.io/cluster-system-anonymous created

//查看pod网络

[root@master ~]#  kubectl get pods -o wide
NAME                    READY   STATUS    RESTARTS   AGE     IP            NODE            NOMINATED NODE
nginx-dbddb74b8-s4rdt   1/1     Running   0          4m18s   172.17.39.3   192.168.20.20   <none>

//在对应网段的node节点上操作可以直接访问

[root@node1 cfg]# curl 172.17.39.3
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

//访问就会产生日志
//回到master01操作

[root@master ~]# kubectl logs nginx-dbddb74b8-s4rdt
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
172.17.39.1 - - [07/Oct/2020:08:31:26 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"