k8s kubedam 搭建方式 （包含高可用解决方法）

#全部服务器安装ansible工具
#ansible 一定要部署在一台性能最好的服务器上面，执行apt update，下载软件的时候需要比较高的性能，不然会卡顿死机
apt install ansible
#生成key
ssh-keygen
#将key文件传给所有的指定主机，免密登陆
ssh-copy-id root@192.168.98.127

#关闭按swap分区
ansible host -m command -a 'sudo swapoff -a' #临时关闭

#开放指定端口
ansible host -m command -a 'usf allow port-id'

#确保时区、时间正确
ansible host -m command -a 'sudo timedatectl set-timezone Asia'
ansible host -m command -a 'sudo timedatectl'

#确保虚机不会自动suspend
ansible host -m command -a 'sudo systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target'


#加载内核模块br_netfilter，并调整参数
ansible host -m command -a 'sudo modprobe br_netfilter'
 ansible host -m command -a  'lsmod | grep br_netfilter'

#调整内核参数，创建k8s.conf，如下：
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-ip6tables = 1

net.bridge.bridge-nf-call-iptables = 1

EOF

#传输到每个服务器
ansible node -m copy -a "src=/etc/sysctl.d/k8s.conf dest=/etc/sysctl.d/k8s.conf backup=yes mode=744"

#使配置生效
ansible host -m command -a  'sudo sysctl --system'

#设置rp_filter的值
sudo vi /etc/sysctl.d/10-network-security.conf
将文件中如下两个参数的值从2修改为1

net.ipv4.conf.default.rp_filter=1

net.ipv4.conf.all.rp_filter=1


ansible node -m copy -a "src=/etc/sysctl.d/10-network-security.conf dest=/etc/sysctl.d/10-network-security.conf backup=yes mode=744"

使配置生效
ansible host -m command -a 'sudo sysctl --system'

#部署核心组件
#安装docker
ansible host -m command -a 'sudo apt update && sudo apt install -y docker.io'

#调整cgroups的驱动

#安装后默认cgroups驱动使用cgroupfs ，需要调整为systemd，因此，编辑docker配置文件，执行：sudo vi /etc/docker/daemon.json

#添加如下内容：

{

"exec-opts": ["native.cgroupdriver=systemd"]

}

ansible node -m copy -a "src=/etc/docker/daemon.json dest=/etc/docker/daemon.json backup=yes mode=744"

#重启docker，执行：

ansible host -m command -a 'sudo systemctl daemon-reload && sudo systemctl restart docker'

#检查当前cgroups驱动，执行：

sudo docker info | grep -i cgroup


#安装k8s组件
#安装k8s组件前的准备


ansible host -m command -a 'sudo apt-get update && sudo apt-get install -y ca-certificates curl software-properties-common apt-transport-https'

#添加k8s源，执行：
curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -



sudo tee /etc/apt/sources.list.d/kubernetes.list <<EOF

deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main

EOF
ansible node -m copy -a "src=/etc/apt/sources.list.d/kubernetes.list dest=/etc/apt/sources.list.d/kubernetes.list backup=yes mode=744"

#安装k8s组件

ansible host -m command -a 'sudo apt update'

ansible host -m command -a 'sudo apt install kubeadm=1.17.4-00 kubelet=1.17.4-00 kubectl=1.17.4-00 -y'

ansible host -m command -a 'sudo apt-mark hold kubelet kubeadm kubectl'

ansible host -m command -a 'sudo systemctl enable kubelet'

#查看镜像需要的版本信息
kubeadm config images list

#在每一台主机上执行脚本
for imageName in ${images[@]};do

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName

docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName k8s.gcr.io/$imageName

docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/$imageName

done
#或者一个个下载也可以
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.17.17
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.17.17 k8s.gcr.io/kube-proxy:v1.17.17
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.17.17 

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.17.17
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.17.17 k8s.gcr.io/kube-apiserver:v1.17.17
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.17.17 



docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17 k8s.gcr.io/kube-controller-manager:v1.17.17
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.17.17 


docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.17.17
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.17.17 k8s.gcr.io/kube-scheduler:v1.17.17
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.17.17 

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1 

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.3-0
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.3-0 


docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.5
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.6.5 


#修改cloud

sudo vim /etc/cloud/cloud.cfg

preserve_hostname: true

ansible node -m copy -a "src=/etc/cloud/cloud.cfg dest=/etc/cloud/cloud.cfg backup=yes mode=744"


#hosts设置

sudo vim /etc/hosts
#添加所有主机地址内容
ansible node -m copy -a "src=/etc/hosts dest=/etc/hosts backup=yes mode=744"

#初始化master节点
#创建集群 --apiserver-advertise-address改为master本机ip

#kubeadm init --apiserver-advertise-address=192.168.33.10 创建必要文件

#创建集群 --apiserver-advertise-address改为master本机ip
kubeadm init --apiserver-advertise-address=192.168.33.10 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12
 --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers --ignore-preflight-errors=all 


mkdir -p $HOME/.kube

sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

sudo chown $(id -u):$(id -g) $HOME/.kube/config--pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12 --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers --ignore-preflight-errors=all


#主节点pod网络设置
#下载配置

wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml



#部署网络

kubectl apply -f kube-flannel.yml
#查看集群状态(Ready)

kubectl get nodes

#新令牌生成方式

kubeadm token create --print-join-command

#安装完成后，kubectl get node 可查看节点状态，由NotReady变成Ready则正常，需要等几分钟完成。
#加入node节点
#在master生成token

kubeadm token create --print-join-command


#在node使用token加入master

kubeadm join 192.168.33.10:6443 --token sqdfvx.89ugmx0kid7wtdt1 --discovery-token-ca-cert-hash sha256:57cce0a7a8d0c4e003596ce320129944b79f58ae23ad7e6a76bd72ab5557839b


#在master查看节点
#加入完成后，在master节点kubectl get nodes可查看已加入的所有节点

高可用解决方法

#首先我们一个 kubeadn 的配置文件，如果一开始安装集群的时候你就是使用的配置文件，那么我们可以直接更新这个配置文件，但是我们你没有使用这个配置文件，直接使用 kubeadm init 来安装的集群，那么我们可以从集群中获取 kubeadm 的配置信息来插件一个配置文件，
#因为 kubeadm 会将其配置写入到 kube-system 命名空间下面的名为 kubeadm-config 的 ConfigMap 中。可以直接执行如下所示的命令将该配置导出：会生成一个 kubeadm.yam的配置文件
kubectl -n kube-system get configmap kubeadm-config -o jsonpath='{.data.ClusterConfiguration}' > kubeadm.yaml

#修改kubeadm.yam,我们这里要添加一个新的域名 api.k8s,local 以及 k8s-master 和 k8s-master2 这两个主机名和IP地址 192.168,.166.128、192.168.166.129、192.168.168.100。
#可以添加多个IP，192.168.166.100为虚拟VIP.
apiServer:
  certSANs:
  - api.k8s.local
  - k8s-master1
  - k8s-master2
  - 192.168.166.128
  - 192.168.166.129
  - 192.168.166.100
  extraArgs:
    authorization-mode: Node,RBAC
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/k8sxio
kind: ClusterConfiguration
kubernetesVersion: v1.17.11
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}

#kubeadm 配置文件后我们就可以更新证书了，首先我们移动现有的 APIServer 的证书和密钥，因为 kubeadm 检测到他们已经存在于指定的位置，它就不会创建新的了
mv /etc/kubernetes/pki/apiserver.{crt,key} ~

#然后直接使用 kubeadm 命令生成一个新的证书：

kubeadm init phase certs apiserver --config kubeadm.yaml
W0902 10:05:28.006627     832 validation.go:28] Cannot validate kubelet config - no validator is available
W0902 10:05:28.006754     832 validation.go:28] Cannot validate kube-proxy config - no validator is available
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [ydzs-master kubernetes kubernetes.default kubernetes.default.svc 
kubernetes.default.svc.cluster.local api.k8s.local k8s-master1 k8s-master2] and IPs [10.96.0.1  192.168.166.128 192.168.166.129 192.168.166.100]
#没出现这一行信息就是证书没有更新，记得先更新证书

#重启 APIServer 来接收新的证书，最简单的方法是直接杀死 APIServer 的容器：
docker ps | grep kube-apiserver | grep -v pause
docker kill docker-id

#验证证书
 openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text


#如果上面的操作都一切顺利，最后一步是将上面的集群配置信息保存到集群的 kubeadm-config 这个 ConfigMap 中去，这一点非常重要，这样以后当我们使用 kubeadm 来操作集群的时候，相关的数据不会丢失，比如升级的时候还是会带上 certSANs 中的数据进行签名的。
kubeadm config upload from-file --config kubeadm.yaml #更新配置

# 如果上面命令报错，可以直接编辑修改 添加需要的内容即可
kubectl -n kube-system edit configmap kubeadm-config

#使用上面的命令保存配置后，我们同样可以用下面的命令来验证是否保存成功了：
kubectl -n kube-system get configmap kubeadm-config -o yaml

#部署 nginx + keepalived 高可用负载均衡器
apt install nginx keepalived -y

#Nginx配置文件（主/备一样）
cat > /etc/nginx/nginx.conf << "EOF"
load_module /usr/lib/nginx/modules/ngx_stream_module.so;  #添加内核
user nginx;  #这里套添加nginx用户
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

# 四层负载均衡，为两台Master apiserver组件提供负载均衡，是要stream要添加内核，在第一行已经添加了参数
stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream k8s-apiserver {
       server 192.168.166.128:6443;   # Master1 APISERVER IP:PORT
       server 192.168.166.129:6443;   # Master2 APISERVER IP:PORT
    }
    
    server {
       listen 16443;  # 由于nginx与master节点复用，这个监听端口不能是6443，否则会冲突
       proxy_pass k8s-apiserver;
    }
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    server {
        listen       80 default_server;
        server_name  _;

        location / {
        }
    }
}
EOF

#keepalived配置文件（Nginx Master）
cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 
    interface ens33  # 修改为实际网卡名
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.166.100/24 #vip地址
    } 
    track_script {
        check_nginx
    } 
}
EOF

#准备上述配置文件中检查nginx运行状态的脚本：
cat > /etc/keepalived/check_nginx.sh  << "EOF"
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF
chmod +x /etc/keepalived/check_nginx.sh


#keepalived配置文件（Nginx Backup）
cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP 
    interface ens33
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90
    advert_int 1
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        192.168.166.100/24 #vip地址
    } 
    track_script {
        check_nginx
    } 
}
EOF

#准备上述配置文件中检查nginx运行状态的脚本：
cat > /etc/keepalived/check_nginx.sh  << "EOF"
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF
chmod +x /etc/keepalived/check_nginx.sh

#启动并设置开机启动
systemctl daemon-reload
systemctl start nginx
systemctl start keepalived
systemctl enable nginx
systemctl enable keepalived

#查看keepalived工作状态
ip add |grep ens33

#Nginx+Keepalived高可用测试
#关闭主节点Nginx，测试VIP是否漂移到备节点服务器。
#在Nginx Master执行 pkill nginx
#在Nginx Backup，ip addr命令查看已成功绑定VIP。

# 修改 kubelet 配置
$ vi /etc/kubernetes/kubelet.conf
......
    server: https://192.168.166.100:16443
  name: kubernetes
......
$ systemctl restart kubelet
# 修改 controller-manager
$ vi /etc/kubernetes/controller-manager.conf
......
    server: https://192.168.166.100:16443
  name: kubernetes
......
# 重启
$ docker kill $(docker ps | grep kube-controller-manager | \
grep -v pause | cut -d' ' -f1)
# 修改 scheduler
$ vi /etc/kubernetes/scheduler.conf
......
    server: https://192.168.166.100:16443
  name: kubernetes
......
# 重启
$ docker kill $(docker ps | grep kube-scheduler | grep -v pause | \
cut -d' ' -f1)


#然后更新 kube-proxy
$ kubectl -n kube-system edit cm kube-proxy
......
  kubeconfig.conf: |-
    apiVersion: v1
    kind: Config
    clusters:
    - cluster:
        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        server: https://api.k8s.local:8443
      name: default
......
#当然还有 kubectl 访问集群的 ~/.kube/config 文件也需要修改。

#更新master节点
kubectl -n kube-system edit configmap kubeadm-config
然后在当前配置文件里面里面添加 controlPlaneEndpoint 属性，用于指定控制面板的负载均衡器的地址。
apiVersion: v1
data:
ClusterConfiguration: |
apiServer:
certSANs:
- api.k8s.local
- k8s-master1
- k8s-master2
- 192.168.166.128
- 192.168.166.129
- 192.168.166.100
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.166.100:16443    # 添加改配置，vip地址
controllerManager: {}

......

#然后需要在 kube-public 命名空间中更新 cluster-info 
#这个 ConfigMap，该命名空间包含一个Kubeconfig 文件，该文件的 server: 一
#行指向单个控制平面节点。只需使用kubectl -n kube-public edit cm cluster-info 更新该 server: 行以指向控制平面的负载均衡器即可。
$ kubectl -n kube-public edit cm cluster-info
......
    server: https://192.168.166.100:16443
  name: ""
......
$  kubectl cluster-info #检查配置是否成功
Kubernetes master is running at https://192.168.166.100:16443
KubeDNS is running at https://api.k8s.local:8443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
KubeDNSUpstream is running at https://api.k8s.local:8443/api/v1/namespaces/kube-system/services/kube-dns-upstream:dns/proxy
Metrics-server is running at https://api.k8s.local:8443/api/v1/namespaces/kube-system/services/https:metrics-server:/proxy

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

#添加master
#在主节点输出这行脚本生成加入master节点的toker
echo "$(kubeadm token create --print-join-command) --control-plane --certificate-key $(kubeadm init phase upload-certs --upload-certs | tail -1)"

root@master:/pp# echo "$(kubeadm token create --print-join-command) --control-plane --certificate-key $(kubeadm init phase upload-certs --upload-certs
 | tail -1)"
W0726 18:46:09.006414   37566 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0726 18:46:09.006478   37566 validation.go:28] Cannot validate kubelet config - no validator is available
I0726 18:46:10.155711   37575 version.go:251] remote version is much newer: v1.24.3; falling back to: stable-1.17
W0726 18:46:11.758691   37575 validation.go:28] Cannot validate kube-proxy config - no validator is available
W0726 18:46:11.758835   37575 validation.go:28] Cannot validate kubelet config - no validator is available
kubeadm join 192.168.98.42:16443 --token e9g4op.dd1xe2utc7p9r41s     --discovery-token-ca-cert-hash sha256:cf427e153ff8eb922428357026b6e1971c80250eef4d174889c398c64bf538b3 
 --control-plane --certificate-key 2b46f6ddd910fc978a6bfec24b6e4cabcdf875065c5989316ebb00192fa46396 #在要添加的master节点上面输入这个toker

#查看etcd
cat /etc/kubernetes/manifests/etcd.yaml

#查看集群是否正常：
kubectl get node

#最后负载均衡访问测试：

#找K8s集群中任意一个节点，使用curl查看K8s版本测试，使用VIP访问：
curl -k https://192.168.98.42:16443/version
{
  "major": "1",
  "minor": "17",
  "gitVersion": "v1.17.4",
  "gitCommit": "8d8aa39598534325ad77120c120a22b3a990b5ea",
  "gitTreeState": "clean",
  "buildDate": "2020-03-12T20:55:23Z",
  "goVersion": "go1.13.8",
  "compiler": "gc",
  "platform": "linux/amd64"
}



#可以正确获取到K8s版本信息，说明负载均衡器搭建正常。该请求数据流程：curl -> vip(nginx) -> apiserver

#通过查看Nginx日志也可以看到转发apiserver IP：
tail /var/log/nginx/k8s-access.log -f
 
192.168.166.130 192.168.166.129:6443 - [13/Jun/2021:15:06:15 +0800] 200 423
192.168.166.130 192.168.166.128:6443 - [13/Jun/2021:15:06:15 +0800] 200 423
192.168.166.130 192.168.166.128:6443 - [13/Jun/2021:15:06:15 +0800] 200 423
192.168.166.130 192.168.166.129:6443 - [13/Jun/2021:15:06:15 +0800] 200 423
192.168.166.130 192.168.166.129:6443 - [13/Jun/2021:15:06:15 +0800] 200 423
#完美