环境规划

k8s-master1 haproxy-1	192.168.100.97
k8s-master2 haproxy-2	192.168.100.98
k8s-master3	192.168.100.99
node1	192.168.100.100
node2	192.168.100.101
VIP/SLB	192.168.10.10

使用haproxy+keepalived的方式做apiserver的高可用搭建 这里不做过多赘述
另外etcd可做外置方式进行配置，详细见etcd高可用集群搭建，做数据高可用性保障

环境准备工作

关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

关闭selinux
# 临时关闭
setenforce 0
# 永久关闭
sed -i 's/enforcing/disabled/' /etc/selinux/config

#禁用swap
swapoff -a #临时关闭
sed -i 's/.*swap.*/#&/' /etc/fstab #永久关闭

#主机时间保持同步
yum install ntpdate -y
# 设置时间同步服务器
ntpdate time.windows.com
#加入crontab
crontab -e
0 1 * * *   /usr/sbin/ntpdate time.windows.com

必备三调参数：开启bridge网桥模式，关闭ipv6协议

cat > kubernetes.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用swap空间，只有当系统OOM时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf

开启ipvs模式，增加pod调度访问效率

modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#! /bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
授权并验证
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash
/etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_contrack_ipv4

安装docker

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
systemctl start docker
systemctl enable docker

添加国内加速源
这里加速源地址根据阿里云账户生成，可以自行生成或添加别的加速源
mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://yourID.mirror.aliyuncs.com"]
  "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"]
}
EOF
systemctl daemon-reload
systemctl restart docker

安装kubernetes

cat <<EOF >/etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 版本根据需求来定
yum install -y kubeadm-1.20.4 kubectl-1.20.4  kubelet-1.20.4

拉取镜像

由于国内镜像拉取不到所以通过国内镜像源拉取
vim k8simages.sh
#!/bin/bash
url=registry.cn-hangzhou.aliyuncs.com/google_containers
# 根据安装的kubeadm版本修改
version=v1.20.4
images=(`kubeadm config images list --kubernetes-version=$version|awk -F '/' '{print $2}'`)
for imagename in ${images[@]} ; do
  docker pull $url/$imagename
  docker tag $url/$imagename k8s.gcr.io/$imagename
  docker rmi -f $url/$imagename
done
sh k8simages.sh
# 这一步也可以不做，在执行初始化时指定镜像仓库地址即可
#参数
--image-repository registry.aliyuncs.com/google_containers

执行初始化

kube-config.yaml 文件

apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
  podSubnet: "10.244.0.0/16"
  serviceSubnet: "10.96.0.0/12"
  dnsDomain: cluster.local
  # vip配置后的apiserver地址
controlPlaneEndpoint: "192.168.10.10:8443"

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
#启用ipvs
featureGates:
  SupportIPVSProxyMode: true
mode: ipvs

kubeadm init --config=kube-config.yaml --upload-certs
#初始化之后会生成两个token 一个是master节点 一个是node节点加入
#master节点加入集群
kubeadm join 192.168.10.10:8443 --token xcrjlj.vjvr04325ieai5n3 \
   --discovery-token-ca-cert-hash sha256:6f99aa1b6f20469638845cb45ee1ba10d2d80c16117bc3b3bbae620c998fb894 \
   --control-plane --certificate-key 20bc19c07155a6683c7054000a5c6b886d597d8a865fcfe4178fe49ad0ae867d
#node节点加入集群
kubeadm join 192.168.10.10:8443 --token 8l9o6r.63sfkw6p0q61m8f3 \
    --discovery-token-ca-cert-hash sha256:34cb0e6c3c7eef4be382d5a7cecc45fce2f3e075a06e5f81740d644d6e7bd4b7 

在服务器上增加配置文件访问集群

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config

集群初始化成功之后加入节点查看节点状态

kubectl get nodes
NAME          STATUS      ROLES                  AGE   VERSION
k8s-master1   NotReady    control-plane,master   1d   v1.20.4
k8s-master2   NotReady    control-plane,master   1d   v1.20.4
k8s-master3   NotReady    control-plane,master   1d   v1.20.4

部署网络插件

#这里使用flannel网络插件 可选网络插件 calico
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

#部署完成之后
kubectl get nodes
NAME          STATUS      ROLES                  AGE   VERSION
k8s-master1   Ready    control-plane,master   1d   v1.20.4
k8s-master2   Ready    control-plane,master   1d   v1.20.4
k8s-master3   Ready    control-plane,master   1d   v1.20.4

#设置master节点可调度 删除污点
kubectl taint node k8s-master1 node-role.kubernetes.io/master-
#恢复master节点污点
kubectl taint nodes k8s-master1 node-role.kubernetes.io/master=true:NoSchedule

token过期之后新增node节点

kubeadm token create --print-join-command

创建admin token

kubectl create serviceaccount k8s-admin -n kube-system
kubectl create clusterrolebinding k8s-admin --clusterrole=cluster-admin --serviceaccount=kube-system:k8s-admin
#查看token
kubectl -n kube-system describe secrets $(kubectl -n kube-system get secret | grep k8s-admin | awk '{print $1}')