环境部署说明

双master节点部署角色如下：
master1 IP地址：192.168.78.55

需求组件：kube-apiserver kube-controller-manager kube-scheduler etcd
master2 IP地址：192.168.78.11

需求组件：kube-apiserver kube-controller-manager kube-scheduler
node1节点 IP地址：192.168.78.66

需求组件：kubelet kube-proxy docker-ce flannel etcd
node2节点 IP地址：192.168.78.77

需求组件：kubelet kube-proxy docker-ce flannel etcd
nginx_1 IP地址：192.168.78.22

需求组件：nginx keepalived
nginx_2 IP地址：192.168.78.33

需求组件：nginx keepalived
VIP IP地址：192.168.78.100

Kubernetes 多节点部署 （基于单节点）

1.拷贝 K8S 工作目录、组件启动脚本至 master02

----master01----
'//首先在master01将kubernetes目录传输至master1'
 scp -r /opt/kubernetes/ root@192.168.78.11:/opt/


'//然后再将master中三个组件的启动脚本kube-apiserver.service、kube-controller-manager.service、kube-scheduler.service传输至master2'
scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.78.11:/usr/lib/systemd/system

'//master02需要etcd证书'

master1节点操作

拷贝master01上已有的etcd证书给master2使用

PS：因为新加入的master中也包含apiserver，在apiserver工作时，也会需要与ETCD进行交互，所以也需要ETCD证书进行认证
scp -r /opt/etcd/ root@192.168.78.11:/opt/

2.master02 修改文件

-- mster2
hostnamectl set-hostname master2
su -

systemctl stop firewalld && systemctl disable firewalld

setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

cd /opt/kubernetes/cfg/

vim kube-apiserver 

KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.78.55:2379,https://192.168.78.66:2379,https://192.168.78.66:2379 \		'//这里为etcd集群IP，先不用改'
--bind-address=192.168.78.11 \		'//修改为自身IP'
--secure-port=6443 \
--advertise-address=192.168.78.11 \		'//修改IP'
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem"		'//注意，这里用到了etcd证书'

#apiserver
systemctl start kube-apiserver.service 
systemctl enable kube-apiserver.service 
systemctl status kube-apiserver.service
#控制器
systemctl start kube-controller-manager.service 
systemctl enable kube-controller-manager.service
systemctl status kube-controller-manager.service
#调度器
systemctl start kube-scheduler.service 
systemctl enable kube-scheduler.service 
systemctl status kube-scheduler.service 

 ps aux|grep kube
'//检查进程，三个组件是否正常启动'

cat >> /etc/profile << EOF 
export PATH=$PATH:/opt/kubernetes/bin/	
EOF	

source /etc/profile 


kubectl get node

'//至此，还没有配置完成，2个node节点只认master01'
'//还需要搭建一个负载均衡群集以完成高可用性，以下使用2个nginx完成'

3.搭建 nginx/keepalived

----nging01、02----
hostnamectl set-hostname nginx01
su -
systemctl stop firewalld && systemctl disable firewalld
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

vim /etc/yum.repos.d/nginx.repo		'//建立nginx的YUM仓库，以便使用yum'

[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/		'//指定URL访问路径'
enabled=1		'//开启此yum源，默认项可省略'
gpgcheck=0		'//不验证软件包的签名'

yum install -y nginx

vim /etc/nginx/nginx.conf		'//配置nginx，添加四层转发'

  9 events {
 10     worker_connections  1024;
 11 }		'//插入以下内容'
 12 stream {
 13 
 14    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';		'//日志格式'
 15     access_log  /var/log/nginx/k8s-access.log  main;		'//K8S日志存放路径'
 16 
 17     upstream k8s-apiserver {		'//配置负载均衡，指向master'
 18         server 192.168.78.55:6443;
 19         server 192.168.78.11:6443;
 20     }
 21     server {
 22                 listen 6443;		'//访问端口'
 23                 proxy_pass k8s-apiserver;		'//转发调动proxy访问代理'
 24     }
 25     }
 26     


nginx -t		'//检查语法'

systemctl start nginx && systemctl enable nginx

 netstat -natp|grep nginx


*************************************************************
----nging01、02----
'//部署keepalived服务'
yum -y install keepalived
vim /etc/keepalived/keepalived.conf 
'//修改配置文件'
! Configuration File for keepalived 
 
global_defs {		'//收邮件地址 '
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   }		'//邮件发送地址 '
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER 
} 

vrrp_script check_nginx {		'//检查nginx的服务脚本，与之相关联'
    script "/etc/nginx/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 				'//注：nginx02为BACKUP'
    interface ens33
    virtual_router_id 51		'VRRP路由ID实例，每个实例是唯一的' 
    priority 100				'优先级，nginx02设置90' 
    advert_int 1				'指定VRRP心跳包通告间隔时间，默认1秒' 
    authentication { 
        auth_type PASS
        auth_pass 123123
    }  
    virtual_ipaddress {
        192.168.78.100/24		'//VIP' 
    } 
    track_script {
        check_nginx				'//检测nginx，触发keepalived'
    } 
}


**************************************************************
----nging01、02----
 vim /etc/nginx/check_nginx.sh
'//count为变量，用于统计'
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
'//过滤nginx进程'
if [ "$count" -eq 0 ];then		'//0表示nginx没有运行'
    systemctl stop keepalived		'//则关闭keepalived'
fi

 chmod +x /etc/nginx/check_nginx.sh  '//赋权'

----nginx01、02---


systemctl start keepalived.service && systemctl enable keepalived.service

 ip a		'//显示网络设备，查看VIP'

inet 192.168.126.100/24 scope global secondary ens33

----nginx02----
 systemctl start keepalived.service && systemctl enable keepalived.service 

systemctl status keepalived.service 
 ip a
'VIP在nginx01中'

'//可在nginx01中关闭nginx服务，再去nginx02使用ip a查看VIP是否漂移'
'//恢复操作：回到nginx01中重新启动nginx与keepalived，VIP就会漂移回来，nginx01优先级比nginx02高'

4.node 节点修改配置文件

----node01、02---
'//开始修改node节点中配置文件，统一为VIP'
 vim /opt/kubernetes/cfg/bootstrap.kubeconfig 
vim /opt/kubernetes/cfg/kubelet.kubeconfig 
 vim /opt/kubernetes/cfg/kube-proxy.kubeconfig 
server: https://192.168.78.100:6443

systemctl restart kubelet.service kube-proxy.service 
cd /opt/kubernetes/cfg/
 grep 100 *		'//过滤检查'


----nginx01----
systemctl restart nginx
tail /var/log/nginx/k8s-access.log		'//查看K8S日志，确认实现负载均衡'

5.master 创建 pod 用以测试

----master01----
'//创建pod用以测试'
 kubectl run nginx --image=nginx

 kubectl get pods		'//等待一会儿查看，处于创建状态，Running为成功'


'//查看pod日志'
kubectl logs nginx-dbddb74b8-rwz94

Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-rwz94)

'//报错，不具备对应权限查看日志，以下为解决办法'
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous

 kubectl logs nginx-dbddb74b8-rwz94

 kubectl describe pod nginx-dbddb74b8-rwz94
'//可进一步查看详细信息'
'//还可以去对应node节点查看docker，已经创建相应容器（基础）'

kubectl get pods -o wide		'//查看所创建pods的IP'

'//192.168.78.66为node01，下一步去node01节点查看'

6.node 节点访问 nginx 页面

----node01----
'//在对应网段的node节点上可以访问nginx页面'
[root@node01 cfg]# curl 172.17.16.3


----master01----
'//node01访问nginx页面后，会产生日志，回到master01查看'
kubectl logs nginx-dbddb74b8-rwz94