#（1）软件包下载

Node包：https://dl.k8s.io/v1.9.6/kubernetes-node-linux-amd64.tar.gz
 cd /tools/node
tar xf kubernetes-node-linux-amd64.tar.gz

#（2）在master给kubelet启动时的用户名授权

kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要先将 bootstrap token 文件中的 kubelet-bootstrap 用户赋予 system:node-bootstrapper 角色，然后 kubelet 才有权限创建认证请求(certificatesigningrequests)。-user=kubelet-bootstrap时token.csv中的用户名, 需要发送到master上; 这里后面传到master; 

在master上运行： kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

#（3）配置kubelet
注意：没有特殊说明, 相关配置都是在跳板机上进行
1）设置集群参数

kubectl config set-cluster kubernetes \
    --certificate-authority=/opt/kubernetes/ssl/ca.pem \
    --embed-certs=true \
    --server=https://192.168.19.128:6443 \
    --kubeconfig=bootstrap.kubeconfig

2)设置客户端认证参数，token同样来自前文master节点上文件token.csv,目前在跳板机上;

     kubectl config set-credentials kubelet-bootstrap \
    --token=9b21fb338b667a07adc604d73c3eaf63 \
    --kubeconfig=bootstrap.kubeconfig

3）设置上下文参数

kubectl config set-context default   --cluster=kubernetes   --user=kubelet-bootstrap   --kubeconfig=bootstrap.kubeconfig

4）设置默认上下文

kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

5）准备kubelet启动脚本文件

cat  >kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/sbin/kubelet \
    --address=0.0.0.0 \
    --hostname-override=0.0.0.0 \
    --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/wangfang-k8s/pause-amd64:3.0 \
    --experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
    --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
    --cert-dir=/opt/kubernetes/ssl \
    --cluster-dns=10.254.0.2\
    --cluster-domain=cluster.local. \
    --hairpin-mode promiscuous-bridge \
    --allow-privileged=true \
    --serialize-image-pulls=false \
    --logtostderr=true \
    --v=2
ExecStartPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT
ExecStartPost=/sbin/iptables -A INPUT -s 172.16.0.0/12 -p tcp --dport 4194 -j ACCEPT
ExecStartPost=/sbin/iptables -A INPUT -s 192.168.0.0/16 -p tcp --dport 4194 -j ACCEPT
ExecStartPost=/sbin/iptables -A INPUT -p tcp --dport 4194 -j DROP
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

准备修改node节点kubelet启动服务文件0.0.0.0的脚本文件

    #cat change_ip.sh
    ip=$(ip addr |grep ens33|grep inet|awk '{print $2}'|awk -F'/' '{print $1}')
    sed -ri "s/=0.0.0.0/=$ip/g" /usr/lib/systemd/system/kubelet.service

5）把相关文件传到node节点

#把kubelet二进制程序传到node节点
ansible node -m copy -a 'src=/tools/node/kubernetes/node/bin/kubelet  dest=/usr/local/sbin/kubelet mode=0755'
#kube-proxy二进制程序传到node节点
ansible node -m copy -a 'src=/tools/node/kubernetes/node/bin/kube-proxy  dest=/usr/local/sbin/kube-proxy mode=0755'
#把kubelet向api-server认证所需要的文件传到node节点
ansible node -m copy -a 'src=bootstrap.kubeconfig  dest=/opt/kubernetes/cfg/bootstrap.kubeconfig'
#准备kubelet工作目录
ansible node -m shell -a 'mkdir /var/lib/kubelet'
#把kubelet服务启动脚本文件传到node节点
ansible node -m copy -a 'src=kubelet.service  dest=/usr/lib/systemd/system/kubelet.service'
#关闭node的swap
ansible node -m shell -a 'swapoff -a'
#发送修改脚本到node节点
ansible node -m script -a 'change_ip.sh'

6）服务启动

systemctl daemon-reload 
systemctl enable kubelet 
systemctl start kubelet 
systemctl status kubelet 

7）在master对csr授权

kubectl get csr 
kubectl   certificate approve node-csr-YLZHzeWk69Dvsv8JJcOFzj6VK9P4xT0MocCvWvQnrvQ

8）在master上进行角色绑定, 因为kubelet是通过用户system:node 和api-server进行通信, 如果没有权限是没法创建pod

 kubectl create clusterrolebinding kubelet-node-clusterbinding --clusterrole=system:node --group=system:nodes

#（4）配置kube-proxy

1）创建kube-proxy证书签名请求

cat >kube-proxy-csr.json<<EOF
{
    "CN": "system:kube-proxy",
    "hosts": [],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Hangzhou",
            "L": "Hangzhou",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

2）生成kube-proxy证书和私钥

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

3）创建 kube-proxy kubeconfig 文件
\cp *.pem /opt/kubernetes/ssl/
设置集群参数

    #kubectl config set-cluster kubernetes \
    --certificate-authority=/opt/kubernetes/ssl/ca.pem \
    --embed-certs=true \
    --server=https://192.168.19.128:6443 \
    --kubeconfig=kube-proxy.kubeconfig

设置客户端参数

 #kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
--client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

设置上下文参数

# kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

设置默认上下文

#kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

4）准备kube-proxy服务启动脚本文件

cat > kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/sbin/kube-proxy --bind-address=0.0.0.0 --hostname-override= --cluster-cidr=172.30.0.0/16 --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig --logtostderr=true --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

5）文件传到node节点

#创建kube-proxy目录
ansible node -m shell -a 'mkdir -p /var/lib/kube-proxy'
#发送kube-proxy证书和私钥文件到node节点
ansible node -m copy -a 'src=kube-proxy.pem dest=/opt/kubernetes/ssl/kube-proxy.pem'
ansible node -m copy -a 'src=kube-proxy-key.pem dest=/opt/kubernetes/ssl/kube-proxy-key.pem'
#发送kubeconfig文件到node节点;
ansible node -m copy -a 'src=kube-proxy.kubeconfig dest=/opt/kubernetes/cfg/kube-proxy.kubeconfig'
#发送服务启动脚本文件到node节点
ansible node -m copy -a 'src=kube-proxy.service dest=/usr/lib/systemd/system/kube-proxy.service'

6）启动服务

systemctl daemon-reload
systemctl start kube-proxy
systemctl enable kube-proxy 
systemctl status kube-proxy

#（5）创建pod资源验证集群是否部署正确

kubectl run nginx-test --image=nginx:1.11 --replicas=3
kubectl get pods -o wide
kubectl expose deployment nginx-test --port=88 --target-port=80 --type=NodePort
kubectl get svc 
kubectl get ep