使用rke安装高可用k8s集群

1. 服务器rke集群节点角色规划

用户	主机名	内网IP	SSH端口	系统	rke 角色
ops	rke-server-01	192.168.2.131	22	CentOS Linux release 7.6.1810 (Core)	controlplane、worker、etcd
ops	rke-server-02	192.168.2.132	22	CentOS Linux release 7.6.1810 (Core)	controlplane、worker、etcd
ops	rke-server-03	192.168.2.133	22	CentOS Linux release 7.6.1810 (Core)	controlplane、worker、etcd

2. 系统准备（所有节点 用root 用户操作）

• 安装一些个人常用的基础安装包

 yum  -y install  epel-release.noarch
 yum  -y install   psmisc gcc gcc-c++ texinfo wget unzip zip gcc libticonv-devel  libcurl-devel curl nmap iotop dstat tree mlocate ntpdate openssh-clients  net-tools vim  ntsysv nmap curl lrzsz  sysstat  libselinux-python   pcre pcre-devel zlib zlib-devel openssl openssl-devel readline-devel bzip2  httpd-devel python-devel python-pip python-setuptools   lsof  sqlite-devel  nscd bind-utils telnet rsync tcpdump expect  nc    ntp  lftp   bash-completion ipset ipvsadm 

• 关闭防火墙

  systemctl stop firewalld
  systemctl stop iptables
  systemctl disable firewalld
  systemctl disable iptables
  
  

• 关闭selinux

```
sed -i 's/SELINUX=enforcing/SELINUX=disabled/'  /etc/sysconfig/selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/'   /etc/selinux/config  
setenforce 0

```

• 三台时间同步要一致

  systemctl start ntpd
  systemctl enable ntpd
  
  

• 关闭swap分区

  swapoff -a
  sed  -i  '/swap/d' /etc/fstab
  mount  -a
  

  • 然后修改/etc/fstab，把swap分区相关的配置注释掉
• 内核参数调整

  cat >> /etc/sysctl.conf <<EOF
  fs.file-max = 2442652
  net.ipv4.ip_local_port_range = 1024     65535
  vm.swappiness=0
  net.ipv4.ip_forward=1
  net.bridge.bridge-nf-call-iptables=1
  net.bridge.bridge-nf-call-ip6tables = 1
  EOF
  sysctl  -p 
  

• 配置资源限制

     sh -c " cat  >>/etc/security/limits.conf <<EOF
     *       soft    nofile          1048576
     *       hard    nofile          1048576
     *       soft     core            unlimited
     *       hard    core            unlimited
     *       soft	    nproc            unlimited
     *       hard	    nproc           unlimited
     EOF"
     sh -c  "cat >> /etc/security/limits.d/20-nproc.conf << EOF
     *  soft    nproc     unlimited
     *  hard    nproc     unlimited
     EOF"
  

• 加载ipvs相关模块
  由于ipvs已经加入到了内核的主干，所以为kube-proxy开启ipvs的前提需要加载以下的内核模块：

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
 #!/bin/bash
 modprobe -- ip_vs
 modprobe -- ip_vs_rr
 modprobe -- ip_vs_wrr
 modprobe -- ip_vs_sh
 modprobe -- nf_conntrack_ipv4
 EOF
 chmod 755 /etc/sysconfig/modules/ipvs.modules 
 bash /etc/sysconfig/modules/ipvs.modules
 lsmod | grep -e ip_vs -e nf_conntrack_ipv4

• 安装docker

yum remove docker  docker-common  docker-selinux  docker-engine  | /bin/true
yum install -y yum-utils device-mapper-persistent-data lvm2           
yum-config-manager --add-repo  https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum  repolist  
yum  install  docker-ce  -y 
mkdir  -p /etc/docker /data/docker
cat > /etc/docker/daemon.json <<EOF
{
   "data-root": "/data/docker",
   "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl enable docker 
systemctl restart docker

• 创建ops 用户并设置密码

useradd ops -G docker
echo "password" | passwd --stdin ops

3. 在rke-server-01 上配置ops 用户可以SSH免密登录到所有主机（包含rke-server-01）

ssh-keygen  -t rsa 
ssh-copy-id -i /home/ops/.ssh/id_rsa.pub ops@192.168.2.131
ssh-copy-id -i /home/ops/.ssh/id_rsa.pub ops@192.168.2.132
ssh-copy-id -i /home/ops/.ssh/id_rsa.pub ops@192.168.2.133

4. 在rke-server-01下载安装rke和kubectl命令

VERSION=v1.2.11 && \
curl -LO https://github.com/rancher/rke/releases/download/${VERSION}/rke_linux-amd64 && \
chmod +x rke_linux-amd64 && \
mv rke_linux-amd64 /usr/local/bin/rke

curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl
chmod 755 kubectl &&  mv  kubectl /usr/local/bin/

5. 在rke-server-01切换到ops用户使用rke 安装k8s 集群

• 使用rke 交互式在当前目录生成创建集群所需要cluster.yml

rke config 
[+] Cluster Level SSH Private Key Path [~/.ssh/id_rsa]: 
[+] Number of Hosts [1]: 3
[+] SSH Address of host (1) [none]: 192.168.2.131
[+] SSH Port of host (1) [22]: 
[+] SSH Private Key Path of host (192.168.2.131) [none]: 
[-] You have entered empty SSH key path, trying fetch from SSH key parameter
[+] SSH Private Key of host (192.168.2.131) [none]: 
[-] You have entered empty SSH key, defaulting to cluster level SSH key: ~/.ssh/id_rsa
[+] SSH User of host (192.168.2.131) [ubuntu]: ops
[+] Is host (192.168.2.131) a Control Plane host (y/n)? [y]: y
[+] Is host (192.168.2.131) a Worker host (y/n)? [n]: y
[+] Is host (192.168.2.131) an etcd host (y/n)? [n]: y
[+] Override Hostname of host (192.168.2.131) [none]: rke-server-01
[+] Internal IP of host (192.168.2.131) [none]: 
[+] Docker socket path on host (192.168.2.131) [/var/run/docker.sock]: 
[+] SSH Address of host (2) [none]: 192.168.2.132
[+] SSH Port of host (2) [22]: 
[+] SSH Private Key Path of host (192.168.2.132) [none]: 
[-] You have entered empty SSH key path, trying fetch from SSH key parameter
[+] SSH Private Key of host (192.168.2.132) [none]: 
[-] You have entered empty SSH key, defaulting to cluster level SSH key: ~/.ssh/id_rsa
[+] SSH User of host (192.168.2.132) [ubuntu]: ops
[+] Is host (192.168.2.132) a Control Plane host (y/n)? [y]: y
[+] Is host (192.168.2.132) a Worker host (y/n)? [n]: y
[+] Is host (192.168.2.132) an etcd host (y/n)? [n]: y
[+] Override Hostname of host (192.168.2.132) [none]: rke-server-02
[+] Internal IP of host (192.168.2.132) [none]: 
[+] Docker socket path on host (192.168.2.132) [/var/run/docker.sock]: 
[+] SSH Address of host (3) [none]: 192.168.2.133
[+] SSH Port of host (3) [22]: 
[+] SSH Private Key Path of host (192.168.2.133) [none]: 
[-] You have entered empty SSH key path, trying fetch from SSH key parameter
[+] SSH Private Key of host (192.168.2.133) [none]: 
[-] You have entered empty SSH key, defaulting to cluster level SSH key: ~/.ssh/id_rsa
[+] SSH User of host (192.168.2.133) [ubuntu]: ops
[+] Is host (192.168.2.133) a Control Plane host (y/n)? [y]: y
[+] Is host (192.168.2.133) a Worker host (y/n)? [n]: y
[+] Is host (192.168.2.133) an etcd host (y/n)? [n]: y
[+] Override Hostname of host (192.168.2.133) [none]: rke-server-03
[+] Internal IP of host (192.168.2.133) [none]: 
[+] Docker socket path on host (192.168.2.133) [/var/run/docker.sock]: 
[+] Network Plugin Type (flannel, calico, weave, canal, aci) [canal]: flannel
[+] Authentication Strategy [x509]: 
[+] Authorization Mode (rbac, none) [rbac]: rabc
[+] Kubernetes Docker image [rancher/hyperkube:v1.20.9-rancher1]: 
[+] Cluster domain [cluster.local]: 
[+] Service Cluster IP Range [10.43.0.0/16]: 
[+] Enable PodSecurityPolicy [n]: 
[+] Cluster Network CIDR [10.42.0.0/16]: 
[+] Cluster DNS Service IP [10.43.0.10]: 
[+] Add addon manifest URLs or YAML files [no]: 

• 使用已定义好的cluster.yml

# If you intened to deploy Kubernetes in an air-gapped environment,
# please consult the documentation on how to configure custom RKE images.
nodes:
- address: 192.168.2.131
  port: "22"
  internal_address: ""
  role:
  - controlplane
  - worker
  - etcd
  hostname_override: rke-server-01
  user: ops
- address: 192.168.2.132
  port: "22"
  role:
  - controlplane
  - worker
  - etcd
  hostname_override: rke-server-02
  user: ops
- address: 192.168.2.133
  port: "22"
  role:
  - controlplane
  - worker
  - etcd
  hostname_override: rke-server-03
  user: ops
services:
  etcd:
    snapshot: true
    creation: 6h
    retention: 24h
network:
  plugin: "flannel" 
  mtu: 0
  options: {}

• 安装k8s集群

rke up --config ~/cluster.yml
耐心等待安装完成就行

• 验证集群

mkidr -p ~/.kube && cp ./kube_config_cluster.yml ~/.kube/config
kubectl get node 
kubectl get pod  -A 

rke 增加和移除节点

  rke up --update-only --config ~/cluster.yml

彻底清理rke节点

cat > rancher/clear.sh << EOF
df -h|grep kubelet |awk -F % '{print $2}'|xargs umount 
rm /var/lib/kubelet/* -rf
rm /etc/kubernetes/* -rf
rm /var/lib/rancher/* -rf
rm /var/lib/etcd/* -rf
rm /var/lib/cni/* -rf

rm -rf /var/run/calico 

iptables -F && iptables -t nat -F

ip link del flannel.1

docker ps -a|awk '{print $1}'|xargs docker rm -f
docker volume ls|awk '{print $2}'|xargs docker volume rm

rm -rf /var/etcd/
rm -rf /run/kubernetes/
docker rm -fv $(docker ps -aq)
docker volume rm  $(docker volume ls)
rm -rf /etc/cni
rm -rf /opt/cni

systemctl restart docker
EOF
rke remove
sh rancher/clear.sh