安装前准备

内核升级包的md5,本人已验证，只要是这个md5值，放心升级
1ea91ea41eedb35c5da12fe7030f4347  kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
01a6da596167ec2bc3122a5f30a8f627  kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
建议是4.17版本之上就好

 echo "172.17.200.40 k8s-master01" | sudo tee -a /etc/hosts
echo "172.17.200.41 k8s-master02" | sudo tee -a /etc/hosts
echo "172.17.200.42 k8s-master03" | sudo tee -a /etc/hosts
echo "172.17.200.43 k8s-node01" | sudo tee -a /etc/hosts
echo "172.17.200.44 k8s-node02" | sudo tee -a /etc/hosts
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y
systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
yum install ntpdate -y
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
echo -e "* soft nofile 65536\n* hard nofile 131072\n* soft nproc 65535\n* hard nproc 655350\n* soft memlock unlimited\n* hard memlock unlimited" | sudo tee -a /etc/security/limits.conf
cd /root
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
cd /root && yum localinstall -y kernel-ml*
grub2-set-default  0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
grubby --default-kernel
然后重启
 

containerd 配置

yum install containerd -y
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

[root@master01 ~]# vim /etc/containerd/config.toml
[plugins]
...
 [plugins."io.containerd.grpc.v1.cri"]
 ...
 # 修改pause镜像地址
 sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"
 ...
 # 配置 systemd cgroup 驱动
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
 ...
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.op
tions]
 ...
 SystemdCgroup = true

换成这种方式修改
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
sed -i 's/sandbox_image = "registry.k8s.io\/pause:3.6"/sandbox_image = "registry.aliyuncs.com\/google_containers\/pause:3.9"/g' /etc/containerd/config.toml
cat /etc/containerd/config.toml  |grep -i sandbox
cat /etc/containerd/config.toml  |grep -i SystemdCgroup
systemctl daemon-reload
systemctl enable containerd --now

内核参数优化

 
 
yum install ipset ipvsadm -y
mkdir /etc/sysconfig/modules -p
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.overcommit_memory = 0
EOF
sysctl -p /etc/sysctl.d/k8s.conf
cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter

安装nerdctl

nerdctl需要使⽤buildkitd来实现镜像的构建，因此拷⻉相关命令和相关的启动程序。
wget https://github.com/containerd/nerdctl/releases/download/v1.5.0/nerdctl-full-1.5.0-linux-amd64.tar.gz
tar -xf nerdctl-full-1.5.0-linux-amd64.tar.gz 
cp bin/nerdctl /usr/local/bin/
cp bin/buildctl bin/buildkitd /usr/local/bin/
cp lib/systemd/system/buildkit.service /usr/lib/systemd/system/
systemctl daemon-reload
systemctl enable buildkit --now

 修改/etc/profile ，新增
export PATH=$PATH:/usr/local/bin

以上是所有机器全部安装

安装好的部分为
内核优化，containerd， 时间同步等服务器基础配置
buildkit

可以做个重启测试，看服务是否能够开机自启动，我用的是centos 7.9系统
uname -a
Linux gcp--test 4.19.12-1.el7.elrepo.x86_64 #1 SMP Fri Dec 21 11:06:36 EST 2018 x86_64 x86_64 x86_64 GNU/Linux
systemctl status containerd
systemctl status buildkit

开始安装

yum install -y kubelet-1.28.0 kubeadm-1.28.0 kubectl-1.28.0

检查版本
kubeadm version
systemctl enable kubelet --now

下载镜像
kubeadm config images pull \
 --image-repository registry.aliyuncs.com/google_containers \
 --kubernetes-version v1.28.0

输出为
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.28.0
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.9
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.5.9-0
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.10.1

初始化，这步骤容易出问题！

这里区分好 pod service 服务器网段别重复了
kubeadm init \
--apiserver-advertise-address="172.17.200.40" \
--control-plane-endpoint="172.17.200.37" \
--apiserver-bind-port=6443 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.28.0 \
--service-cidr=10.96.0.0/16 \
--pod-network-cidr=10.244.0.0/16 \
--upload-certs \
--service-dns-domain=fly.local

正确输出末尾部分
To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 172.17.200.37:6443 --token fosyex.07pp3s1zd8pqk1qr \
        --discovery-token-ca-cert-hash sha256:a70a555d55967cd210568049518ce5bb7f09fa3221d268a3af8c2 \
        --control-plane --certificate-key 0d268a3af8c20d268a3af8c20d268a3af8c20d268a3af8c20d268a3af8c2

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 172.17.200.37:6443 --token fosyex.07pp3s1zd8pqk1qr \
        --discovery-token-ca-cert-hash sha256:a70a555d55967cd210568049518ce5bb7f09fa3221d268a3af8c2


其他节点添加到这个集群中来,此时会是NotReady ，因为coredns 还不能用
 

安装flannel

wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

 根据情况进行修改
 containers:
 - args:
 - --ip-masq
 - --kube-subnet-mgr
 - --iface=eth0 # 指明绑定在哪个⽹卡上（可不配置）
 net-conf.json: |
 {
 "Network": "10.244.0.0/16",
 "Backend": {
 "Type": "vxlan"
 }
 }

结果展示

[root@gcp-hongkong-k8s-master01-test install_k8s]# kubectl get pod -A
NAMESPACE      NAME                                                     READY   STATUS    RESTARTS       AGE
kube-flannel   kube-flannel-ds-5mtx7                                    1/1     Running   0              27s
kube-flannel   kube-flannel-ds-64fln                                    1/1     Running   0              27s
kube-flannel   kube-flannel-ds-lvqhq                                    1/1     Running   0              27s
kube-flannel   kube-flannel-ds-mwmbx                                    1/1     Running   0              27s
kube-flannel   kube-flannel-ds-pp7w7                                    1/1     Running   0              27s
kube-system    coredns-66f779496c-4xfbc                                 1/1     Running   0              135m
kube-system    coredns-66f779496c-h4hmd                                 1/1     Running   0              135m
kube-system    etcd-gcp-hongkong-k8s-master01-test                      1/1     Running   0              135m
kube-system    etcd-gcp-hongkong-k8s-master02-test                      1/1     Running   0              132m
kube-system    etcd-gcp-hongkong-k8s-master03-test                      1/1     Running   0              132m
kube-system    kube-apiserver-gcp-hongkong-k8s-master01-test            1/1     Running   0              135m
kube-system    kube-apiserver-gcp-hongkong-k8s-master02-test            1/1     Running   0              132m
kube-system    kube-apiserver-gcp-hongkong-k8s-master03-test            1/1     Running   1 (132m ago)   132m
kube-system    kube-controller-manager-gcp-hongkong-k8s-master01-test   1/1     Running   1 (132m ago)   135m
kube-system    kube-controller-manager-gcp-hongkong-k8s-master02-test   1/1     Running   0              132m
kube-system    kube-controller-manager-gcp-hongkong-k8s-master03-test   1/1     Running   0              131m
kube-system    kube-proxy-7vbk2                                         1/1     Running   0              132m
kube-system    kube-proxy-95kvh                                         1/1     Running   0              131m
kube-system    kube-proxy-d47m7                                         1/1     Running   0              131m
kube-system    kube-proxy-nvkjg                                         1/1     Running   0              131m
kube-system    kube-proxy-wnxqp                                         1/1     Running   0              135m
kube-system    kube-scheduler-gcp-hongkong-k8s-master01-test            1/1     Running   1 (132m ago)   135m
kube-system    kube-scheduler-gcp-hongkong-k8s-master02-test            1/1     Running   0              132m
kube-system    kube-scheduler-gcp-hongkong-k8s-master03-test            1/1     Running   0              132m