K8S/Kubernetes

现在稳定性差不多了。可以总结一下了。

真正使用时，有几个地方，还是确认一下，再正式运行吧。

 

#!/bin/bash
# Version V0.09 2019-05-10-10:32

if [ `whoami` != "root" ];then echo "[error] You need to switch to root user to execute this command" ; exit 1 ;fi

K8S_VERSION="1.14.1"
#定义执行kubeadm的普通用户
General_user="xxx"

K8S_VER=1.14.1
#当前目录是在cmd下，所以要向上提一级cd..
dir_path=$(cd `dirname $0`;cd ../;pwd)
cmd_path=$dir_path/cmd
cert_path=$dir_path/cert
rpm_path=$dir_path/rpm

RED_COLOR='\E[1;31m'
GREEN_COLOR='\E[1;32m'
YELOW_COLOR='\E[1;33m'
BLUE_COLOR='\E[1;34m'
PINK='\E[1;35m'
RES='\E[0m'

#如果存在已有文件，先删除，便于反复部署
function if_file_exist_del() {
  if [ -e $1 ]; then
    rm -f $1
  fi
}

env_setting(){
  echo -e "${PINK}***** $FUNCNAME *****${RES}"
  #禁用相关服务
  systemctl stop firewalld.service
  systemctl disable firewalld.service
  setenforce 0
  sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/sysconfig/selinux 
  sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config 
  sed -i "s/^SELINUX=permissive/SELINUX=disabled/g" /etc/sysconfig/selinux 
  sed -i "s/^SELINUX=permissive/SELINUX=disabled/g" /etc/selinux/config
  swapoff -a
  sed -i 's/.*swap.*/#&/' /etc/fstab
  
  #清空iptables
  iptables -P INPUT ACCEPT
  iptables -P FORWARD ACCEPT
  iptables -P OUTPUT ACCEPT
  iptables -t nat -P PREROUTING ACCEPT
  iptables -t nat -P POSTROUTING ACCEPT
  iptables -t nat -P OUTPUT ACCEPT
  iptables -t mangle -P PREROUTING ACCEPT
  iptables -t mangle -P OUTPUT ACCEPT
  iptables -F
  iptables -t nat -F
  iptables -t mangle -F
  iptables -X
  iptables -t nat -X
  iptables -t mangle -X
  
  #修改内核参数
  k8s_kernel_conf=/etc/sysctl.d/k8s.conf
  if_file_exist_del $k8s_kernel_conf
  cat<<EOF >$k8s_kernel_conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
vm.swappiness=0
EOF

  sysctl -p
  sysctl --system
  
  # 安装必须系统软件包并加载模块
  yum install bridge-utils ipset ipvsadm sysstat libseccomp conntrack  conntrack-tools socat -y


  modprobe br_netfilter
  modprobe -- ip_vs
  modprobe -- ipip
  modprobe -- tun
  modprobe -- ip_vs_rr
  modprobe -- ip_vs_wrr
  modprobe -- ip_vs_sh
  modprobe -- nf_conntrack_ipv4
  modprobe -- nf_conntrack_ipv6
  
  # 自动加载模块
  ipvs_no=$(cat /etc/rc.local|grep ip_vs|wc -l)
  if [ $ipvs_no -eq 0 ]; then
      echo "modprobe br_netfilter" >> /etc/rc.local
      echo "modprobe -- ip_vs" >> /etc/rc.local
      echo "modprobe -- ipip" >> /etc/rc.local
      echo "modprobe -- tun" >> /etc/rc.local
      echo "modprobe -- ip_vs_rr" >> /etc/rc.local
      echo "modprobe -- ip_vs_wrr" >> /etc/rc.local
      echo "modprobe -- ip_vs_sh" >> /etc/rc.local
      echo "modprobe -- nf_conntrack_ipv4" >> /etc/rc.local
      echo "modprobe -- nf_conntrack_ipv6" >> /etc/rc.local
  fi  
  
  # sudo命令，这里不能写变量，所以最佳实现应该是写一个demo用户，接着替换成指定用户。灵活。
  k8s_sudoers_conf=/etc/sudoers.d/k8s_sudoers
  if_file_exist_del $k8s_sudoers_conf
   cat<<EOF >$k8s_sudoers_conf
xxx ALL = (root) NOPASSWD:/bin/systemctl restart docker
xxx ALL = (root) NOPASSWD:/bin/systemctl reload docker
xxx ALL = (root) NOPASSWD:/bin/systemctl daemon-reload
xxx ALL = (root) NOPASSWD:/bin/systemctl start kubelet
xxx ALL = (root) NOPASSWD:/bin/systemctl stop docker
xxx ALL = (root) NOPASSWD:/bin/systemctl start docker
xxx ALL = (root) NOPASSWD:/bin/systemctl status docker
xxx ALL = (root) NOPASSWD:/bin/systemctl stop kubelet
xxx ALL = (root) NOPASSWD:/bin/systemctl restart kubelet
xxx ALL = (root) NOPASSWD:/bin/systemctl status kubelet
xxx ALL = (root) NOPASSWD:/usr/sbin/ipvsadm
xxx ALL = (root) NOPASSWD:/usr/bin/docker
xxx ALL = (root) NOPASSWD:/usr/local/bin/kubeadm
xxx ALL = (root) NOPASSWD:/usr/local/bin/kubectl
xxx ALL = (root) NOPASSWD:/usr/bin/chown -R docker /etc/kubernetes/
EOF
}


init_kube(){
  echo -e "${PINK}***** $FUNCNAME *****${RES}"
    # 先清空所有容器，并去除挂载点
  systemctl stop kubelet.service
  docker ps |grep -v "CONTAINER ID"|awk '{print $1}'|xargs -I {} docker stop {}
  docker ps -a|grep -v "CONTAINER ID"|awk '{print $1}'|xargs -I {} docker rm {}
  systemctl stop docker.service
  sleep 30
  for i in $(df|awk '$6 ~ /.*kubelet.*/{print $6}');do
    umount $i
  done
    
    # 便于反复安装，先清除目录和应用
  rm -rf /etc/cni/
  rm -rf /opt/cni/bin/*
  ifconfig docker0 down
  ip link delete docker0
  rm -f /usr/local/bin/kube*
  rm -f /usr/bin/kube*
  
  # 如果有calico网络，这样更顺利
  calico_net_conf=/etc/NetworkManager/conf.d/calico.conf
  if_file_exist_del $calico_net_conf
  cat<<EOF >$calico_net_conf
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF

  # 重新pki，方便证书管理
  pki_dir=/etc/kubernetes
  mkdir -p ${pki_dir}
  rm -rf ${pki_dir}/* 
  chown -R ${General_user}.docker ${pki_dir}
  chmod -R 755 ${pki_dir}

    # 删除再重装rpm包
  yum remove kubeadm -y
  yum remove kubectl -y
  yum remove kubelet -y
  yum localinstall $rpm_path/*.rpm -y  --skip-broken
  chown -R $(id -u ${General_user}):$(id -g ${General_user}) /etc/systemd/system/kubelet*
  # 不知为什么，读的是这个目录，要cp一下。
  /bin/cp /usr/bin/kube* /usr/local/bin/
 
     # 清除可能安装过的网络接口和路由表，干净
  ifconfig -a|grep  -vE '(^[[:space:]]|^$)'|grep -E '(veth|flannel|kube|cni|dummy)'|awk -F ":" '{print $1}'|awk '{for(i=1;i<=NF;i++){print "ip link set " $i " down";}}'|sh
  ifconfig -a|grep  -vE '(^[[:space:]]|^$)'|grep -E '(veth|flannel|kube|cni|dummy)'|awk -F ":" '{print $1}'|awk '{for(i=1;i<=NF;i++){print "ip link delete " $i;}}'|sh
  ip route|grep 10.244|awk '{print $1}'|awk '{for(i=1;i<=NF;i++){print "ip route delete " $i;}}'|sh

  modprobe -r ipip
  modprobe -r ip_gre
  modprobe  ipip
    
    # 自定义pauce，不然要去google下。
  kubelet_sysconfig=/etc/sysconfig/kubelet
  if_file_exist_del $kubelet_sysconfig
  cat<<EOF >$kubelet_sysconfig
KUBELET_EXTRA_ARGS="--pod-infra-container-image=harbor.xxx.cn/3rd_part/k8s.gcr.io/pause:3.1"
EOF

    # 重新加载，完成。
  systemctl daemon-reload
  systemctl start docker
  systemctl enable kubelet && systemctl restart kubelet

  echo -e "${GREEN_COLOR}***** k8s root init system success ******${RES}"
}



function main(){
  env_setting
  init_kube
  kubeadm reset -f
  ipvsadm -C
}
main