#!/bin/bash

#指定kubeasz版本,并下载
release=3.1.1
##k8s集群名称
K8S_CLUSTER_NAME="k8s-gan"
#harbor地址
HARBOR_ADDR=10.0.0.84
#harbor账号
HARBOR_USERNAME="admin"
#harbor密码
HARBOR_PASSWD="123456"
#全部k8s节点
TOTAL_ADDR="10.0.0.81 10.0.0.82 10.0.0.83"
#master节点,不为IP时注意添加解析
MASTER_ADDR="10.0.0.81"
#worker节点
WORKER_ADDR="10.0.0.81 10.0.0.82 10.0.0.83"
#etcd节点
ETCD_ADDR="10.0.0.81 10.0.0.82 10.0.0.83"
#daemon.json复制节点
COPY_ADDR="10.0.0.82"
#docker版本
DOCKER_VER=19.03.15
#k8s版本
K8S_BIN_VER=v1.21.5
#集群网络设置
CLUSTER_NETWORK="calico"
#service网段
SERVICE_CIDR="10.100.0.0/16"
#pod网段
CLUSTER_CIDR="10.200.0.0/16"
#nodePort 端口范围
NODE_PORT_RANGE="30000-65000"
#cluster DNS domain
CLUSTER_DNS_DOMAIN="${K8S_CLUSTER_NAME}.local"
#二进制文件路径
bin_dir="/usr/bin"
#node节点最大pod 数
MAX_PODS=500
#是否开启calico的ipip网络
CALICO_IPV4POOL_IPIP="Always"
#准备工作，安装ansible，关闭防火墙，做免密认证
yum install -y epel-release
yum install -y ansible sshpass
PASS=1234567
#rm -rf /root/.ssh
if [ ! -e /root/.ssh/id_rsa ];then
 ssh-keygen -P "" -f /root/.ssh/id_rsa
fi


for n in ${TOTAL_ADDR};do
sshpass -p $PASS ssh-copy-id -o StrictHostKeyChecking=no  root@${n}:
done

#cat <<EOF >> /etc/hosts
#10.0.0.81  k8s-1
#10.0.0.82  k8s-2
#10.0.0.83  k8s-3
#EOF
for n in $TOTAL_ADDR;do
  { 
#   scp -o StrictHostKeyChecking=no /etc/hosts root@$n:/etc/hosts
   ssh root@${n} sed -i -r "s/^SELINUX=/SELINUX=disabled/" /etc/selinux/config
   ssh root@${n} setenforce 0
   ssh root@${n} systemctl disable --now firewalld
   ssh root@${n} yum makecache && yum update -y 
  }&
done
wait

#curl -C- -fLO --retry 3 https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
#指定docker和k8s版本并修改ezdown文件执行
sed -i "/^DOCKER_VER/c DOCKER_VER=${DOCKER_VER}" ezdown
sed -i "/^K8S_BIN_VER/c K8S_BIN_VER=${K8S_BIN_VER}" ezdown

chmod +x ezdown
./ezdown -D
cd /etc/kubeasz/
./ezctl new ${K8S_CLUSTER_NAME}
#---------------镜像地址---------------------
#calico版本
calico_ver=`sed -r -n "s/^calico_ver:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
sandbox_ver=`sed -r -n "s#SANDBOX_IMAGE:.*\".*:(.*)\"#\1#p" /etc/kubeasz/clusters/k8s-gan/config.yml`
SANDBOX_IMAGE="${HARBOR_ADDR}/baseimages/pause-amd64:${sandbox_ver}"
CALICO_CNI_IMAGE="${HARBOR_ADDR}/baseimages/calico-cni:${calico_ver}"
CALICO_FLEXVOL_IMAGE="${HARBOR_ADDR}/baseimages/calico-pod2daemon-flexvol:${calico_ver}"
CALICO_NODE_IMAGE="${HARBOR_ADDR}/baseimages/calico-node:${calico_ver}"
CALICO_CONTROLLER_IMAGE="${HARBOR_ADDR}/baseimages/calico-kube-controllers:${calico_ver}"
#-------dns镜像---------------------------
corednsVer=`sed -r -n "s/^corednsVer:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
dnsNodeCacheVer=`sed -r -n "s/^dnsNodeCacheVer:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
COREDNS_IMAGE="${HARBOR_ADDR}/baseimages/coredns:${corednsVer}"
NODELOCALDNS_IMAGE="${HARBOR_ADDR}/baseimages/k8s-dns-node-cache:${dnsNodeCacheVer}"
#---------metric-server镜像-------------
metricsVer=`sed -r -n "s/^metricsVer:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
METRIC_SERVER_IMAGE="${HARBOR_ADDR}/baseimages/metrics-server:${metricsVer}"
#---------dashboard镜像-------------
dashboardVer=`sed -r -n "s/^dashboardVer:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
dashboardMetricsScraperVer=`sed -r -n "s/^dashboardMetricsScraperVer:[[:space:]]+\"(.*)\"/\1/p" /etc/kubeasz/clusters/k8s-gan/config.yml`
DASHBOARD_IMAGE="${HARBOR_ADDR}/baseimages/dashboard:$dashboardVer"
METRICS_SCRAPER_IMAGE="${HARBOR_ADDR}/baseimages/metrics-scraper:$dashboardMetricsScraperVer"
#----------------------------------------------




cd /etc/kubeasz/clusters/${K8S_CLUSTER_NAME}
#修改hosts文件的节点地址
sed -n -r  '1,/\[etcd\]/p;/# master node/,$p' hosts  > /tmp/hosts
cat /tmp/hosts > hosts
sed -n -r  '1,/\[kube_master\]/p;/# work node/,$p' hosts > /tmp/hosts
cat /tmp/hosts > hosts
sed -n -r  '1,/\[kube_node\]/p;/\[optional\] harbor server/,$p' hosts > /tmp/hosts
cat /tmp/hosts > hosts
for i in $ETCD_ADDR;do
   sed -i -r  "/\[etcd\]/a $i" hosts
done
for i in $MASTER_ADDR;do
   sed -i -r  "/\[kube_master\]/a $i" hosts
done
for i in $WORKER_ADDR;do
   sed -i -r  "/\[kube_node\]/a $i" hosts
done
#修改hosts文件集群网络
sed -i -r "s%^CLUSTER_NETWORK.*%CLUSTER_NETWORK=\"${CLUSTER_NETWORK}\"%" hosts
#修改service网段
sed -i -r "s%^SERVICE_CIDR.*%SERVICE_CIDR=\"${SERVICE_CIDR}\"%" hosts
#修改pod网段
sed -i -r "s%^CLUSTER_CIDR.*%CLUSTER_CIDR=\"${CLUSTER_CIDR}\"%" hosts
#修改nodePort 端口范围
sed -i -r "s%^NODE_PORT_RANGE.*%NODE_PORT_RANGE=\"${NODE_PORT_RANGE}\"%" hosts
#cluster DNS domain
sed -i -r "s%^CLUSTER_DNS_DOMAIN.*%CLUSTER_DNS_DOMAIN=\"${CLUSTER_DNS_DOMAIN}\"%" hosts
#二进制文件路径
sed -i -r "s%^bin_dir.*%bin_dir=${bin_dir}%" hosts
#不启用容器仓库镜像
sed -i -r "s%^ENABLE_MIRROR_REGISTRY.*%ENABLE_MIRROR_REGISTRY: false%" config.yml
#添加docker信任的harbor仓库
sed -i -r "s%^INSECURE_REG.*%INSECURE_REG: \'[\"127.0.0.1\/8\",\"${HARBOR_ADDR}\"]\'%" config.yml
#修改node节点最大pod数
sed -i -r "s%^MAX_PODS.*%MAX_PODS: ${MAX_PODS}%" config.yml
#打开为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
sed -i -r "s%^KUBE_RESERVED_ENABLED.*%KUBE_RESERVED_ENABLED: \"yes\"%" config.yml
#修改baseimage
sed -i -r "s%^SANDBOX_IMAGE.*%SANDBOX_IMAGE: \"${SANDBOX_IMAGE}\"%" config.yml
#添加docker版本
#sed -i -r  "/DOCKER_STORAGE_DIR/a DOCKER_VER: ${DOCKER_VER}" config.yml
#是否开启calico的ipip网络
sed -i -r "s%^CALICO_IPV4POOL_IPIP.*%CALICO_IPV4POOL_IPIP: \"${CALICO_IPV4POOL_IPIP}\"%" config.yml
#修改calico版本
sed -i -r "s%^calico_ver:.*%calico_ver: \"${calico_ver}\"%" config.yml
#修改/etc/kubeasz/playbooks/01.prepare.yml将ex_lb去掉
sed -i -r "s/^[[:space:]]*(- ex_lb.*)/#\1/" /etc/kubeasz/playbooks/01.prepare.yml
cd /etc/kubeasz
#第一步集群初始化
./ezctl setup ${K8S_CLUSTER_NAME} 01
#第二步安装etcd
./ezctl setup ${K8S_CLUSTER_NAME} 02
#第三步安装docker
./ezctl setup ${K8S_CLUSTER_NAME} 03
#从别的节点拷贝daemon.json到部署节点
scp $COPY_ADDR:/etc/docker/daemon.json /etc/docker/ 
systemctl restart docker 
yum install -y expect

expect <<EOF
set timeout 20
spawn docker login $HARBOR_ADDR
expect  "Username:" { send "${HARBOR_USERNAME}\n" }
expect  "Password:" { send "${HARBOR_PASSWD}\n" }
expect eof
EOF
#上传镜像
SANDBOX_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "easzlab/pause-amd64"`
docker tag $SANDBOX_IMAGE_OLD $SANDBOX_IMAGE && docker push $SANDBOX_IMAGE
CALICO_CNI_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "calico/cni"`
docker tag $CALICO_CNI_IMAGE_OLD $CALICO_CNI_IMAGE && docker push $CALICO_CNI_IMAGE
CALICO_FLEXVOL_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "calico/pod2daemon-flexvol"`
docker tag $CALICO_FLEXVOL_IMAGE_OLD $CALICO_FLEXVOL_IMAGE && docker push $CALICO_FLEXVOL_IMAGE
CALICO_NODE_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "calico/node"`
docker tag $CALICO_NODE_IMAGE_OLD $CALICO_NODE_IMAGE && docker push $CALICO_NODE_IMAGE
CALICO_CONTROLLER_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "calico/kube-controllers"`
docker tag $CALICO_CONTROLLER_IMAGE_OLD $CALICO_CONTROLLER_IMAGE && docker push $CALICO_CONTROLLER_IMAGE
COREDNS_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "coredns/coredns"`
docker tag $COREDNS_IMAGE_OLD $COREDNS_IMAGE && docker push $COREDNS_IMAGE
NODELOCALDNS_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "easzlab/k8s-dns-node-cache"`
docker tag $NODELOCALDNS_IMAGE_OLD $NODELOCALDNS_IMAGE && docker push $NODELOCALDNS_IMAGE
METRIC_SERVER_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "easzlab/metrics-server"`
docker tag $METRIC_SERVER_IMAGE_OLD $METRIC_SERVER_IMAGE && docker push $METRIC_SERVER_IMAGE
DASHBOARD_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "kubernetesui/dashboard"`
docker tag $DASHBOARD_IMAGE_OLD $DASHBOARD_IMAGE && docker push $DASHBOARD_IMAGE
METRICS_SCRAPER_IMAGE_OLD=`docker images | awk 'NR>1{print $1":"$2}' | grep -v "${HARBOR_ADDR}" |grep "kubernetesui/metrics-scraper"`
docker tag $METRICS_SCRAPER_IMAGE_OLD $METRICS_SCRAPER_IMAGE && docker push $METRICS_SCRAPER_IMAGE


#第四步添加master
./ezctl setup ${K8S_CLUSTER_NAME} 04
#第五步添加worker
./ezctl setup ${K8S_CLUSTER_NAME} 05
cd /etc/kubeasz/roles/calico/templates
calico_ver_simple=`echo $calico_ver |sed -r "s/(v[0-9]{1}\.[0-9]{1,2})\.[0-9]{1,2}/\1/"`
sed -i -r  "s%image: docker.io\/calico\/cni.*%image: ${CALICO_CNI_IMAGE}%"  calico-${calico_ver_simple}.yaml.j2
sed -i -r  "s%image: docker.io\/calico\/pod2daemon-flexvol.*%image: ${CALICO_FLEXVOL_IMAGE}%"  calico-${calico_ver_simple}.yaml.j2
sed -i -r  "s%image: docker.io\/calico\/node.*%image: ${CALICO_NODE_IMAGE}%"  calico-${calico_ver_simple}.yaml.j2
sed -i -r  "s%image: docker.io\/calico\/kube-controllers.*%image: ${CALICO_CONTROLLER_IMAGE}%"  calico-${calico_ver_simple}.yaml.j2
cd /etc/kubeasz
#第六步安装calico网络
./ezctl setup ${K8S_CLUSTER_NAME} 06
#修改coredns 资源限制和修改为本地镜像
cd /etc/kubeasz/roles/cluster-addon/templates/dns
sed -i -r "s%image: coredns\/coredns.*%image: ${COREDNS_IMAGE}%" coredns.yaml.j2
sed -i -r "s%image: easzlab\/k8s-dns-node-cache.*%image: ${NODELOCALDNS_IMAGE}%" nodelocaldns-iptables.yaml.j2
sed -i -r "s%image: easzlab\/k8s-dns-node-cache.*%image: ${NODELOCALDNS_IMAGE}%" nodelocaldns-ipvs.yaml.j2
sed -i -r "s%replicas: 1%replicas: 3%" coredns.yaml.j2
sed -i -r "/limits/a  cpu: 4" coredns.yaml.j2
sed -i -r "s%^(cpu: 4)%            \1%" coredns.yaml.j2
sed -i -r "s%memory: 200Mi%memory: 4096Mi%" coredns.yaml.j2
sed -i -r "s%cpu: 100m%cpu: 500m%" coredns.yaml.j2
sed -i -r "s%memory: 70Mi%memory: 512Mi%" coredns.yaml.j2
sed -i -r "s%forward \. /etc/resolv\.conf%forward \. 223\.5\.5\.5%" coredns.yaml.j2
sed -i -r "s%max_concurrent 1000%max_concurrent 5000%" coredns.yaml.j2
#修改metric-server为本地镜像
cd /etc/kubeasz/roles/cluster-addon/templates/metrics-server
sed -i -r "s%image: easzlab\/metrics-server.*%image: ${METRIC_SERVER_IMAGE}%" components.yaml.j2
#修改dashboard为本地镜像
cd /etc/kubeasz/roles/cluster-addon/templates/dashboard
sed -i -r "s%image: kubernetesui\/dashboard.*%image: ${DASHBOARD_IMAGE}%" kubernetes-dashboard.yaml.j2
sed -i -r "s%image: kubernetesui\/metrics-scraper.*%image: ${METRICS_SCRAPER_IMAGE}%" kubernetes-dashboard.yaml.j2
sed -i -r "s%--token-ttl=1800%--token-ttl=3600%" kubernetes-dashboard.yaml.j2
sed -i -r "/targetPort: 8443/a nodePort: 38443" kubernetes-dashboard.yaml.j2
sed -i -r "s%^(nodePort: 38443.*)%      \1%" kubernetes-dashboard.yaml.j2
cd /etc/kubeasz
##第七步安装额外插件(dns、dashboard、metric-server)
./ezctl setup ${K8S_CLUSTER_NAME} 07
#创建etcd备份定时任务
echo  "* 1 * * *   /etc/kubeasz/ezctl backup ${K8S_CLUSTER_NAME} &>/dev/null" | crontab
#生成dashboard的kubeconfig并传回到桌面
admin_user_token=`kubectl get secret -A  | grep admin | awk '{print $2}'`
token=`kubectl describe  secret  ${admin_user_token} -n kube-system`
token=`echo ${token} | sed -n  -r "s%.*token:[[:space:]]+([^[:space:]]+).*%\1%p"`
cp /root/.kube/config /tmp/kubeconfig
echo -e "    token: ${token}" >> /tmp/kubeconfig
yum -y install lrzsz
sz /tmp/kubeconfig