一、master节点安装

master节点需要的三个主要组件分别是kube-apiserver，kube-controller-manager，kube-scheduler。

• kube-apiserver作为最核心的大脑，接收集群外请求，根据请求调度k8s所有组件工作并最终把数据存储在etcd中。
• kube-controller-manager是控制器的管理器，控制器确保各资源在node节点上按需运行，控制器管理器确保控制器正常运行。
• kube-scheduler选取最优的node节点处理kube-apiserver接收到的请求。

1.1证书准备

与kube-apiserver的请求交互https进行更安全且是双向认证，即客户端需要认证apiserver，而apiserver也需要认证客户端，所以我们需要额外准备一套证书。
构建私有ca

mkdir /opt/k8s_ca
cd /opt/k8s_ca/
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json <<EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

构建apiserver的证书

cat > server-csr.json <<EOF
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "192.168.80.112",
      "192.168.80.130",
      "192.168.80.146",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

构建kube-proxy证书
用于kube-apiserver和kube-proxy的双向认证

cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

构建admin证书
用于kube-apiserver和k8s管理员(向k8s集群发出请求)的双向认证

cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

1.2安装kube-apiserver组件

mkdir /opt/kubernetes/{bin,cfg,ssl} -p
cp /opt/k8s_ca/*pem  /opt/kubernetes/ssl
wget https://img.yiyao.cc/kubernetes-server-linux-amd64.tar
tar xf kubernetes-server-linux-amd64.tar
cp kube-apiserver kube-scheduler kube-controller-manager kubectl /opt/kubernetes/bin

#创建token文件，用于TLS bootstrapping 功能，让kubelet先使用一个预定的低权限用户连接到 apiserver，然后向 apiserver申请证书。
##更具体的内容可参看文档：https://mritd.me/2018/01/07/kubernetes-tls-bootstrapping-note/
cat > /opt/kubernetes/cfg/token.csv <<EOF
bd41d77ac7cad4cfaa27f6403b1ccf16,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

vim /opt/kubernetes/cfg/kube-apiserver
	KUBE_APISERVER_OPTS="--logtostderr=false \
	--log-dir=/var/log/kubernetes/apiserver \
	--v=3 \
	--etcd-servers=https://192.168.80.112:2379,https://192.168.80.130:2379,https://192.168.80.146:2379 \
	--bind-address=192.168.80.112 \
	--secure-port=6443 \
	--advertise-address=192.168.80.112 \
	--allow-privileged=true \
	--service-cluster-ip-range=10.0.0.0/16 \
	--enable-admission-plugins=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota,NodeRestriction \
	--authorization-mode=RBAC,Node \
	--enable-bootstrap-token-auth \
	--token-auth-file=/opt/kubernetes/cfg/token.csv \
	--service-node-port-range=30000-50000 \
	--tls-cert-file=/opt/kubernetes/ssl/server.pem  \
	--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
	--client-ca-file=/opt/kubernetes/ssl/ca.pem \
	--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
	--kubelet-client-certificate=/opt/kubernetes/ssl/admin.pem \
	--kubelet-client-key=/opt/kubernetes/ssl/admin-key.pem \
	--etcd-cafile=/opt/etcd/ssl/ca.pem \
	--etcd-certfile=/opt/etcd/ssl/server.pem \
	--etcd-keyfile=/opt/etcd/ssl/server-key.pem"

vim /usr/lib/systemd/system/kube-apiserver.service
	[Unit]
	Description=Kubernetes API Server
	Documentation=https://github.com/kubernetes/kubernetes
	
	[Service]
	EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver
	ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS
	Restart=on-failure
	
	[Install]
	WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

1.3安装kube-controller-manager组件

vim /opt/kubernetes/cfg/kube-controller-manager
	KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \
	--log-dir=/var/log/kubernetes/controller-manager \
	--v=3 \
	--master=127.0.0.1:8080 \
	--leader-elect=true \
	--address=127.0.0.1 \
	--service-cluster-ip-range=10.0.0.0/16 \
	--cluster-cidr=10.10.0.0/16 \
	--cluster-name=kubernetes \
	--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \
	--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \
	--root-ca-file=/opt/kubernetes/ssl/ca.pem \
	--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem"

vim /usr/lib/systemd/system/kube-controller-manager.service
	[Unit]
	Description=Kubernetes Controller Manager
	Documentation=https://github.com/kubernetes/kubernetes
	 
	[Service]
	EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager
	ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
	Restart=on-failure
	
	[Install]
	WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager

1.4安装kube-scheduler组件

vim /opt/kubernetes/cfg/kube-scheduler
	KUBE_SCHEDULER_OPTS="--logtostderr=true \
	--v=4 \
	--master=127.0.0.1:8080 \
	--leader-elect"
vim /usr/lib/systemd/system/kube-scheduler.service
	[Unit]
	Description=Kubernetes Scheduler
	Documentation=https://github.com/kubernetes/kubernetes
	
	[Service]
	EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler
	ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
	Restart=on-failure
	
	[Install]
	WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler

1.5验证master组件的状态

[root@192-168-80-112 k8s_ca]# kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}

二、node节点安装

2.1为TLS bootstrapping创建用户

在master为TLS bootstrapping创建用户

/opt/kubernetes/bin/kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

2.2为TLS bootstrapping创建连接apiserver配置文件

cd /opt/k8s_ca/
KUBE_APISERVER="https://192.168.80.112:6443"
BOOTSTRAP_TOKEN=bd41d77ac7cad4cfaa27f6403b1ccf16

#向bootstrap.kubeconfig文件注入有关kube-apiserver的认证的参数
/opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=./ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig

#向bootstrap.kubeconfig文件注入上下文，关于认证使用的用户和集群名
/opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig

#设置kubeconfig使用的默认用户
/opt/kubernetes/bin/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

2.3为kube-proxy创建连接apiserver配置文件

#向kube-proxy.kubeconfig文件注入有关对集群api-server的认证
/opt/kubernetes/bin/kubectl config set-cluster kubernetes \
--certificate-authority=./ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig

#向kube-proxy.kubeconfig文件注入有关kube-proxy作为客户端被api-server认证
/opt/kubernetes/bin/kubectl config set-credentials kube-proxy \
--client-certificate=./kube-proxy.pem \
--client-key=./kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig

#向kube-proxy.kubeconfig文件注入上下文，关于认证使用的用户和集群名
/opt/kubernetes/bin/kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

#设置kubeconfig使用的默认用户
/opt/kubernetes/bin/kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

scp *kubeconfig root@192.168.80.130:/opt/kubernetes/cfg
scp *kubeconfig root@192.168.80.146:/opt/kubernetes/cfg

2.4安装docker

yum install -y https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/containerd.io-1.2.5-3.1.el7.x86_64.rpm 
yum install -y https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-18.09.6-3.el7.x86_64.rpm
yum install -y https://mirrors.aliyun.com/docker-ce/linux/centos/7/x86_64/stable/Packages/docker-ce-cli-18.09.6-3.el7.x86_64.rpm

#为了方便镜像的拉取，修改使用国内阿里的源
[root@192-168-80-130 ~]# cat /etc/docker/daemon.json 
{
  "registry-mirrors": ["https://afel9vbj.mirror.aliyuncs.com"]
}
systemctl daemon-reload
systemctl enable docker
systemctl start docker
systemctl status docker

2.5安装kubelet组件

scp kubelet kube-proxy root@192.168.80.130:/opt/kubernetes/bin
vim /opt/kubernetes/cfg/kubelet
	KUBELET_OPTS="--logtostderr=false \
	--log-dir=/tmp/kubelet \
	--v=3 \
	--hostname-override=k8snode1 \
	--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
	--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
	--config=/opt/kubernetes/cfg/kubelet.config \
	--cert-dir=/opt/kubernetes/ssl \
	--client-ca-file=/opt/kubernetes/ssl/ca.pem \
	--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"

vim /opt/kubernetes/cfg/kubelet.config
	kind: KubeletConfiguration
	apiVersion: kubelet.config.k8s.io/v1beta1
	address: 192.168.80.130
	port: 10250
	readOnlyPort: 10255
	cgroupDriver: cgroupfs
	clusterDNS: ["10.0.0.2"]
	clusterDomain: cluster.local.
	failSwapOn: false
	authentication:
	  anonymous:
	    enabled: true
	  webhook:
	    enabled: false

vim /usr/lib/systemd/system/kubelet.service
	[Unit]
	Description=Kubernetes Kubelet
	After=docker.service
	Requires=docker.service
	
	[Service]
	EnvironmentFile=/opt/kubernetes/cfg/kubelet
	ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS
	Restart=on-failure
	KillMode=process
	
	[Install]
	WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable kubelet    
systemctl start kubelet    
systemctl status kubelet 

2.6master批准node节点

/opt/kubernetes/bin/kubectl get csr
/opt/kubernetes/bin/kubectl certificate approve node-csr-SJvxb_b-sBqfsLo-ILzaQm_6S9DH_w3THpLGCLNkLDU

2.7安装kube-proxy

vim /opt/kubernetes/cfg/kube-proxy
	KUBE_PROXY_OPTS="--logtostderr=false \
	--log-dir=/tmp/kube-proxy \
	--bind-address=192.168.80.130 \
	--v=3 \
	--cluster-cidr=10.10.0.0/16 \
	--masquerade-all \
	--feature-gates=SupportIPVSProxyMode=true \
	--proxy-mode=ipvs \
	--ipvs-min-sync-period=5s \
	--ipvs-sync-period=5s \
	--ipvs-scheduler=rr \
	--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig"
	
vim /usr/lib/systemd/system/kube-proxy.service
	[Unit]
	Description=Kubernetes Proxy
	After=network.target
	
	[Service]
	EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy
	ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS
	Restart=on-failure
	
	[Install]
	WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable kube-proxy
systemctl start kube-proxy
systemctl status kube-proxy

#安装ipvsadm命令，用于观察kube-proxy代理的状况
ipvsadm -L  -n