1、初始化脚本

#!/bin/bash
swapoff -a
sed -i -r '/swap/s/^/#/' /etc/fstab
systemctl stop firewalld.service
systemctl disable firewalld.service
setenforce 0
sed -ri 's/SELINUX=permissive/SELINUX=disabled/' /etc/selinux/config
cat /etc/selinux/config  | grep -w "SELINUX"
mkdir /var/lib/etcd
mkdir /var/lib/docker
yum -y install chrony
#systemctl enable ntpd
#systemctl start ntpd
sed -i -e '/^server/s/^/#/' -e '1a server time1.cloud.tencent.com iburst' /etc/chrony.conf
systemctl restart chronyd.service
timedatectl set-timezone Asia/Shanghai
cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_nonlocal_bind = 1
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

sysctl -p /etc/sysctl.d/k8s.conf
ulimit -c 0 && echo 'ulimit -S -c 0' >>/etc/profile
modprobe br_netfilter && modprobe iptable_nat && echo iptable_nat >> /etc/modules-load.d/iptable.conf
sed -i 's/crashkernel=auto/& cgroup.memory=nokmem transparent_hugepage=never/g' /etc/default/grub
grub2-mkconfig -o /boot/grub2/grub.cfg
#grub2-mkconfig -o /boot/efi/EFI/kylin/grub.cfg

2、安装HA高可用

2.1、安装Keepalived、haproxy

yum install keepalived haproxy -y

2.2、修改配置文件

cd /etc/keepalived
cat keepalived.conf

! Configuration File for keepalived
global_defs {
  router_id LVS_DEVEL
script_user root
  enable_script_security
}
vrrp_script chk_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 5
  weight -5
  fall 2 
rise 1
}
vrrp_instance VI_1 {
  state MASTER
  interface ens34
  mcast_src_ip 10.10.10.11
  virtual_router_id 51
  priority 100
  advert_int 2
  authentication {
      auth_type PASS
      auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
      10.10.10.19
  }
  track_script {
     chk_apiserver
  }
}

! Configuration File for keepalived
global_defs {
  router_id LVS_DEVEL
script_user root
  enable_script_security
}
vrrp_script chk_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 5
  weight -5
  fall 2 
rise 1
}
vrrp_instance VI_1 {
  state MASTER
  interface ens34
  mcast_src_ip 10.10.10.12
  virtual_router_id 51
  priority 99
  advert_int 2
  authentication {
      auth_type PASS
      auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
      10.10.10.19
  }
  track_script {
     chk_apiserver
  }
}

! Configuration File for keepalived
global_defs {
  router_id LVS_DEVEL
script_user root
  enable_script_security
}
vrrp_script chk_apiserver {
  script "/etc/keepalived/check_apiserver.sh"
  interval 5
  weight -5
  fall 2 
rise 1
}
vrrp_instance VI_1 {
  state MASTER
  interface ens34
  mcast_src_ip 10.10.10.13
  virtual_router_id 51
  priority 98
  advert_int 2
  authentication {
      auth_type PASS
      auth_pass K8SHA_KA_AUTH
  }
  virtual_ipaddress {
      10.10.10.19
  }
  track_script {
     chk_apiserver
  }
}

Keepalived检测脚本

#!/bin/bash
VIRTUAL_IP=10.10.10.19
VIRTUAL_PORT=8888

errorExit() {
    echo "*** $*" 1>&2
    exit 1
}

if ip addr | grep -q $VIRTUAL_IP ; then
    curl -s --max-time 2 --insecure https://${VIRTUAL_IP}:${VIRTUAL_PORT}/healthz  -o /dev/null || errorExit "Error GET https://${VIRTUAL_IP}:${VIRTUAL_PORT}/healthz"
else
    exit 1
fi

haproxy配置文件

# /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    log /dev/log local0
    log /dev/log local1 notice
    daemon

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 1
    timeout http-request    10s
    timeout queue           20s
    timeout connect         5s
    timeout client          20s
    timeout server          20s
    timeout http-keep-alive 10s
    timeout check           10s


#---------------------------------------------------------------------
# apiserver frontend which proxys to the masters
#---------------------------------------------------------------------
frontend apiserver
    bind *:8888
    mode tcp
    option tcplog
    default_backend apiserver

#---------------------------------------------------------------------
# round robin balancing for apiserver
#---------------------------------------------------------------------
backend apiserver
    option httpchk GET /healthz
    http-check expect status 200
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
        server master01 10.10.10.11:6443 check
        server master02 10.10.10.12:6443 check
        server master03 10.10.10.13:6443 check
        # [...]
        # hostname ip:prot 按需更改

2.3、启动服务，开机自启

systemctl start keepalived.service haproxy.service && systemctl enable keepalived.service haproxy.service

3、安装runtime

3.1、安装docker

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum -y install docker-ce
mkdir /etc/docker
cat > /etc/docker/daemon.json << EOF
{
  "insecure-registries": ["http://10.10.10.10"],
  "live-restore": true,
  "registry-mirrors": ["https://2lc8pjse.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF
systemctl enable docker && systemctl start docker
docker --version

##添加http私有仓库添加配置："insecure-registries": ["http://10.10.10.10"]
##添加docker服务重启不影响运行中的容器添加配置："live-restore": true,

3.2、安装containerd

containerd相关入门教程：https://www.modb.pro/db/100271

# 配置docker源3步
# step 1: 安装必要的一些系统工具
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
# Step 2: 添加软件源信息
sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# Step 3 配置yum源
sudo sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# 安装containerd
yum install -y containerd
# 启动containerd
systemctl enable containerd --now
# 查看containerd状态
systemctl status containerd

# 新建目录/etc/containerd
mkdir /etc/containerd
# 生成containerd配置文件
containerd config default > /etc/containerd/config.toml

#对应模块修改配置
vi /etc/containerd/config.toml
...
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.6"
...
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
          endpoint = ["https://b9pmyelo.mirror.aliyuncs.com"]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
          endpoint = ["https://registry.aliyuncs.com/k8sxio"]
         
...
SystemdCgroup = true
                   
#netdctl命令行工具基本命令和docker对标
wget -c https://github.com/containerd/nerdctl/releases/download/v0.22.0/nerdctl-0.22.0-linux-amd64.tar.gz
tar xf nerdctl-0.22.0-linux-amd64.tar.gz
cp nerdctl /usr/bin/
nerdctl version

4、安装kubeadm，kubelet和kubectl

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubelet-1.22.8 kubeadm-1.22.8 kubectl-1.22.8
systemctl enable kubelet

4.1、导出配置根据自己的环境修改

kubeadm config print init-defaults > kubeadm-config.yaml #获取默认配置文件
kubeadm config images pull --config kubeadm-config.yaml  #下载相关镜像
kubeadm init --config kubeadm-config.yaml --upload-certs				 #初始化集群

4.1.1、kubeadm配置文件示例

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.10.10.11
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  imagePullPolicy: IfNotPresent
  name: k8s-master01
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "10.10.10.19:8888"
controllerManager: {}
dns:
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.22.8
networking:
  dnsDomain: cluster.local
  podSubnet: "10.244.0.0/16"
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

5、部署calico

#下载calico配置文件
wget https://docs.projectcalico.org/manifests/calico.yaml
pod的ip段改为kubeadm-config.yaml设置的podSubnet的ip段
  # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
            - name: CALICO_IPV4POOL_CIDR
              value: "10.244.0.0/16"
            # Disable file logging so `kubectl

kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml

6、部署dashboard

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.1/aio/deploy/recommended.yaml
#官网dashboard默认最低权限，要是需要集群管理建议使用管理员权限

默认Dashboard只能集群内部访问，需要修改Service为NodePort类型

7、命令补全

yum install -y bash_completion
source /usr/share/bash-completion/bash_completion
echo 'source <(kubectl completion bash)' >>~/.bashrc

8、k8s扩容

ps：操作master重新纳管节点需要在原有etcd集群中移除

#查看etcd集群节点
ETCDCTL_API=3 etcdctl --endpoints 127.0.0.1:2379 --cacert /etc/kubernetes/pki/etcd/ca.crt --cert  /etc/kubernetes/pki/etcd/server.crt --key  /etc/kubernetes/pki/etcd/server.key member list
member list节点列表
member remove 删除节点
查看节点状态
etcdctl --endpoints=https://10.10.10.12:2379,https://10.10.10.13:2379,https://10.10.10.11:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key endpoint health

8.1 初始化要扩容的k8s节点（如系统设置、安装runtime、安装kubeadm、安装高可用等）

8.2、 在master上生成新的token

kubeadm token create --print-join-command
kubeadm join 10.10.10.19:8888 --token zbpo3b.45eqoh53ucz8qu9e --discovery-token-ca-cert-hash sha256:795fab9145b4f405cc08f10c5ff1d5e37b873ba612136b15b565e5f4cf30911e

8.3、生成新证书

kubeadm init phase upload-certs --upload-certs
I0128 14:32:56.118715   71067 version.go:255] remote version is much newer: v1.26.1; falling back to: stable-1.22
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
29ddf4c384f786fb737b06dd3ce57985dc3a9b5befbce8fd0a1ce2c63e724df9
#新的证书就是29ddf4c384f786fb737b06dd3ce57985dc3a9b5befbce8fd0a1ce2c63e724df9

8.4、扩容master节点

kubeadm join 10.10.10.19:8888 --token zbpo3b.45eqoh53ucz8qu9e --discovery-token-ca-cert-hash sha256:795fab9145b4f405cc08f10c5ff1d5e37b873ba612136b15b565e5f4cf30911e --control-plane --certificate-key 29ddf4c384f786fb737b06dd3ce57985dc3a9b5befbce8fd0a1ce2c63e724df9

8.5、扩容node节点

kubeadm join 10.10.10.19:8888 --token zbpo3b.45eqoh53ucz8qu9e --discovery-token-ca-cert-hash sha256:795fab9145b4f405cc08f10c5ff1d5e37b873ba612136b15b565e5f4cf30911e

9、etcd备份

###### 从master节点上etcd容器内获得 etcdctl 二进制文件 #####
docker cp 3522dc7affc8:/usr/local/bin/etcdctl /usr/bin
###创建备份目录
mkdir /opt/etcd_back
###### 将如下内容放入新文件etcd_back.sh 备份文件保存周期：10天 #####
#!/bin/bash
IP=127.0.0.1
BACKUP=/opt/etcd_back
export ETCDCTL_API=3
etcdctl --endpoints=https://$IP:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save $BACKUP/snap-$(date +%Y%m%d%H%M).db

find /opt/etcd_back -mtime +5 -name "snap-*"|xargs rm -rf

##### 加入定时任务，每8小时备份一次 #####
echo "0 */8 * * * root bash /opt/etcd_back/etcd_bak.sh" >> /etc/crontab