问题

k8s node节点加入到集群时卡住 “[preflight] Running pre-flight checks”

# master节点重新生成加入命令
kubeadm token create --ttl 0 --print-join-command

参考

注意

k8s1.24+使用containerd而不再使用docker，因此使用k8s1.23版本

环境

k8s: 1.23.0
ubuntu: 20.04
docker: 23
docker-compose: 1.25.4

主机	IP	角色
xcrj01	192.168.66.10	k8s-master01
xcrj02	192.168.66.20	k8s-node01
xcrj03	192.168.66.21	k8s-node02

hyper-v ubuntu20.04

参考

hyper-v koolshare

参考

docker

参考
修改daemon.json

{
  "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","http://hub-mirror.c.163.com","http://registry.docker-cn.com"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}

创建systemd所需目录

mkdir -p /etc/systemd/system/docker.service.d

docker-compose

参考

k8s

master

# 切换root用户
sudo su -

# 更新源
apt update
# 升级包
apt upgrade -y

# 设置hostname，可选
# hostnamectl set-hostname k8s-master01

# 备份hosts文件
cp /etc/hosts /etc/hosts-bk
# 添加hosts，见下
vim /etc/hosts

# 开启IPVS负载均衡，见下

# 设置k8s所需内核参数，见下
vim /etc/sysctl.d/kubernetes.conf
# 应用内核参数
sysctl --system

# docker
# 见大标题docker

# 永久关闭swap。pod运行在swap分区中会大大影响效率
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 禁用SELINUX：开启会限制服务进程访问资源
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
# 安装必要包
apt install -y curl gnupg2 software-properties-common apt-transport-https ca-certificates
# 开启路由功能
sysctl -w net.ipv4.ip_forward=1
# 重启虚拟机系统

# kubernetes
# add aliyun/kubernetes apt key
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
# add aliyun/Kubernetes apt repository
apt-add-repository "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
# 检查更新
apt update
# 安装kubelet kubeadm kubectl
apt install -y kubelet=1.23.0-00 kubeadm=1.23.0-00 kubectl=1.23.0-00
# 锁定版本，不随apt upgrade更新
apt-mark hold kubelet kubeadm kubectl
# 查看kubeadm版本
kubeadm version
# 生成初始化文件
kubeadm config print init-defaults > kubeadm-config.yaml
# 修改初始化文件，见下
# 初始化kubernetes
kubeadm init --config=kubeadm-config.yaml --v=6 --upload-certs | tee kubeadm-init.log

# 下面的命令来自 保存打印的内容，推荐使用普通用户
su xcrj01
## 普通用户执行这个，To start using your cluster, you need to run the following as a 
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
## root用户执行这个，Alternatively, if you are the root user, you can run
export KUBECONFIG=/etc/kubernetes/admin.conf

# 配置CNI网络
# You should now deploy a pod network to the cluster. 安装calico网络组件
# kubectl apply -f "https://docs.projectcalico.org/manifests/calico.yaml"
# 暂时使用Flannel（基于L2）构建扁平化的网络，以后升级到Calico（完全基于L3）
sudo kubectl apply -f "https://github.com/flannel-io/flannel/blob/master/Documentation/kube-flannel.yml"

hosts

   #master01
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.20 k8s-node01
   192.168.66.21 k8s-node02
   
   #node01
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.20 k8s-node01
   
   #node02
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.21 k8s-node02

开启IPVS负载均衡

modprobe br_netfilter

或

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#操作上面创建的文件
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

kubernetes.conf

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

kubeadm-config.yaml
增加podSubnet；修改kubernetesVersion；修改advertiseAddress（masterIP）；修改imageRepository为阿里云

   apiVersion: kubeadm.k8s.io/v1beta2
   bootstrapTokens:
   - groups:
     - system:bootstrappers:kubeadm:default-node-token
     token: abcdef.0123456789abcdef
     ttl: 24h0m0s
     usages:
     - signing
     - authentication
   kind: InitConfiguration
   localAPIEndpoint:
     advertiseAddress: 192.168.66.10
     bindPort: 6443
   nodeRegistration:
     criSocket: /var/run/dockershim.sock
     name: k8s-master01
     taints:
     - effect: NoSchedule
       key: node-role.kubernetes.io/master
   ---
   apiServer:
     timeoutForControlPlane: 4m0s
   apiVersion: kubeadm.k8s.io/v1beta2
   certificatesDir: /etc/kubernetes/pki
   clusterName: kubernetes
   controllerManager: {}
   dns:
     type: CoreDNS
   etcd:
     local:
       dataDir: /var/lib/etcd
   imageRepository: registry.aliyuncs.com/google_containers
   kind: ClusterConfiguration
   kubernetesVersion: v1.23.0
   networking:
     dnsDomain: cluster.local
     podSubnet: 10.244.0.0/16
     serviceSubnet: 10.96.0.0/12
   scheduler: {}

node

# 切换root用户
sudo su -

# 更新源
apt update
# 升级包
apt upgrade -y

# 设置hostname，可选
# hostnamectl set-hostname k8s-master01

# 备份hosts文件
cp /etc/hosts /etc/hosts-bk
# 添加hosts，见下
vim /etc/hosts

# 开启IPVS负载均衡，见下

# 设置k8s所需内核参数，见下
vim /etc/sysctl.d/kubernetes.conf
# 应用内核参数
sysctl --system

# docker
# 见大标题docker

# 永久关闭swap。pod运行在swap分区中会大大影响效率
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 禁用SELINUX：开启会限制服务进程访问资源
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
# 安装必要包
apt install -y curl gnupg2 software-properties-common apt-transport-https ca-certificates
# 开启路由功能
sysctl -w net.ipv4.ip_forward=1
# 重启虚拟机系统

# kubernetes
# add aliyun/kubernetes apt key
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
# add aliyun/Kubernetes apt repository
apt-add-repository "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
# 检查更新
apt update
# 安装kubelet kubeadm kubectl
apt install -y kubelet=1.23.0-00 kubeadm=1.23.0-00 kubectl=1.23.0-00
# 锁定版本，不随apt upgrade更新
apt-mark hold kubelet kubeadm kubectl
# 查看kubeadm版本
kubeadm version

# 下面的命令来自 保存打印的内容，推荐使用普通用户
su xcrj02
# 来自kubeadm-init.log文件，192.168.66.10 masterIP
sudo kubeadm join 192.168.66.10:6443 --token xxcbh2.xdulqkbvvrup3b90 \
        --discovery-token-ca-cert-hash sha256:b84434080d676bf402aba832343faf07f119b2d261b95440ff47d67fd4d78eee

hosts

   #master01
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.20 k8s-node01
   192.168.66.21 k8s-node02
   
   #node01
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.20 k8s-node01
   
   #node02
   vi /etc/hosts
   192.168.66.10 k8s-master01
   192.168.66.21 k8s-node02

开启IPVS负载均衡

modprobe br_netfilter

或

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#操作上面创建的文件
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

kubernetes.conf

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1