参考前辈博客及官网：
https://kubernetes.io/zh/docs/reference/setup-tools/kubeadm/kubeadm/
https://www.codercto.com/a/61023.html
https://www.kubernetes.org.cn/5462.html

calicocli安装使用参考
https://docs.projectcalico.org/v3.8/getting-started/calicoctl/configure/kdd

前言

先来张官方kubeadmin HA架构图，帮助理解

一、基础环境配置

1.设置主机名hostname

2.编辑 /etc/hosts 文件，添加域名解析，并规划角色。

cat <<EOF >>/etc/hosts

192.168.30.133 openshift1
192.168.30.129 openshift2 keepalive haproxy
192.168.30.134 openshift3
192.168.30.136 openshift-node

EOF

3.关闭防火墙、selinux和swap。

systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
swapoff -a
sed -i 's/.*swap.*/#&/' /etc/fstab

4.配置内核参数，将桥接的IPv4流量传递到iptables的链

cat > /etc/sysctl.d/k8s.conf <<EOF

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness=0

EOF

sysctl --system

5.加载ipvs模块（手动拷贝进去）

cat /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_modules_dir="/usr/lib/modules/`uname -r`/kernel/net/netfilter/ipvs"
for i in `ls $ipvs_modules_dir | sed  -r 's#(.*).k.*$#\1#'`; do
    /sbin/modinfo -F filename $i  &> /dev/null
    if [ $? -eq 0 ]; then
        /sbin/modprobe $i
    fi
done

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

5.配置国内yum源

yum install -y wget
mkdir /etc/yum.repos.d/bak && mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.cloud.tencent.com/repo/centos7_base.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.cloud.tencent.com/repo/epel-7.repo
yum clean all && yum makecache

配置国内Kubernetes源

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

EOF

配置 docker 源

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

二、部署keepalived和haproxy（对应规划主机）

1. 安装keepalived和haproxy

yum install -y keepalived haproxy

2. 修改配置
   keepalived配置
   openshift2的priority为100，openshift3的priority为90，其他配置一样，这里vip设置为192.168.30.88。

[root@openshift2 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
        zzd@163.com
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_1
}

vrrp_instance VI_1 {
    state MASTER          
    interface ens33
    lvs_sync_daemon_inteface ens33
    virtual_router_id 88
    advert_int 1
    priority 100         
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
      192.168.30.88/24
    }
}

haproxy配置
openshift2和openshift3的haproxy配置是一样的。此处我们监听的是192.168.30.88的8443端口，因为haproxy是和k8s apiserver是部署在同一台服务器上，都用6443会冲突。

[root@openshift2 ~]# cat /etc/haproxy/haproxy.cfg
global
        chroot  /var/lib/haproxy
        daemon
        group haproxy
        user haproxy
        log 127.0.0.1:514 local0 warning
        pidfile /var/lib/haproxy.pid
        maxconn 20000
        spread-checks 3
        nbproc 8

defaults
        log     global
        mode    tcp
        retries 3
        option redispatch

listen https-apiserver
        bind 192.168.30.88:8443
        mode tcp
        balance roundrobin
        timeout server 900s
        timeout connect 15s

        server apiserver01 192.168.30.133:6443 check port 6443 inter 5000 fall 5
        server apiserver02 192.168.30.129:6443 check port 6443 inter 5000 fall 5
        server apiserver03 192.168.30.134:6443 check port 6443 inter 5000 fall 5

3. 启动服务

systemctl enable keepalived && systemctl start keepalived 
systemctl enable haproxy && systemctl start haproxy 

三、软件安装

注：在所有节点上进行如下操作

1.安装docker，并做优化

yum install -y docker-ce-18.06.1.ce-3.el7
systemctl enable docker && systemctl start docker
docker --version


cat <<EOF > /etc/docker/daemon.json 
{
  "exec-opts": ["native.cgroupdriver=systemd"]
}

EOF

systemctl restart docker

2.安装kubeadm、kubelet、kubectl（如果安装指定版本加上版本号，默认 yum install -y kubelet kubeadm kubectl ipvsadm 安装最新版本）

yum install -y kubelet-1.14.4 kubectl-1.14.4 kubeadm-1.14.4 ipvsadm
systemctl enable kubelet

四、 配置kubeadmin初始化文件（部署master上执行）

参考：https://kubernetes.io/zh/docs/reference/setup-tools/kubeadm/kubeadm-init/

重要参数说明：
advertiseAddress是本机master地址
controlPlaneEndpoint是LB地址（如果是单maser这里就写一样）
podSubnet 这个参数没啥用，最后会被cni插件替换
serviceSubnet service地址范围
dataDir etcd数据存储到外挂磁盘上

# cat kubeadm-init.yaml 
apiVersion: kubeadm.k8s.io/v1beta1
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.30.133
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: openshift1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
apiServer:
  timeoutForControlPlane: 4m0s
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.30.88:8443"
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kubernetesVersion: v1.14.4
networking:
  dnsDomain: cluster.local
  podSubnet: "10.145.0.0/16"
  serviceSubnet: "10.245.0.0/16"
scheduler: {}
controllerManager: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"

4.3预先下载镜像

kubeadm config images pull --config kubeadm-init.yaml

4.4初始化集群

[root@openshift1 ~]# kubeadm init --config kubeadm-init.yaml 
[init] Using Kubernetes version: v1.14.4
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [openshift1 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.245.0.1 192.168.30.133 192.168.30.88]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [openshift1 localhost] and IPs [192.168.30.133 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [openshift1 localhost] and IPs [192.168.30.133 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 20.011015 seconds
[upload-config] storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.14" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --experimental-upload-certs
[mark-control-plane] Marking the node openshift1 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node openshift1 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] creating the "cluster-info" ConfigMap in the "kube-public" namespace
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities 
and service account keys on each node and then running the following as root:

  kubeadm join 192.168.30.88:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:61ed61986ca4e382544a30d368cd974dfffe1e2e0eb7c831c0d7bd9e1db2ac7a \
    --experimental-control-plane 	  

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.30.88:8443 --token 3h5nnp.vmxh3hjttl4ss2bx \
    --discovery-token-ca-cert-hash sha256:61ed61986ca4e382544a30d368cd974dfffe1e2e0eb7c831c0d7bd9e1db2ac7a

注意：kubeadm init生成的token有效期只有1天，如果你的node节点在使用kubeadm join时出现如下错误，请到master上检查你所使用的token是否有效
[preflight] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml’
error execution phase preflight: unable to fetch the kubeadm-config ConfigMap: failed to get config map: Unauthorized

生成不过期的token（后续加入使用不过期的token替换默认生成的，即EXPIRES=never）
kubeadm token create --ttl 0 --print-join-command

[root@openshift1 ~]# kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
3h5nnp.vmxh3hjttl4ss2bx authentication,signing system:bootstrappers:kubeadm:default-node-token
abcdef.0123456789abcdef 22h 2019-08-22T16:49:33+08:00 authentication,signing system:bootstrappers:kubeadm:default-node-token

kubeadm init主要执行了以下操作：
[init]：指定版本进行初始化操作
[preflight] ：初始化前的检查和下载所需要的Docker镜像文件
[kubelet-start] ：生成kubelet的配置文件”/var/lib/kubelet/config.yaml”，没有这个文件kubelet无法启动，所以初始化之前的kubelet实际上启动失败。
[certificates]：生成Kubernetes使用的证书，存放在/etc/kubernetes/pki目录中。
[kubeconfig] ：生成 KubeConfig 文件，存放在/etc/kubernetes目录中，组件之间通信需要使用对应文件。
[control-plane]：使用/etc/kubernetes/manifest目录下的YAML文件，安装 Master 组件。
[etcd]：使用/etc/kubernetes/manifest/etcd.yaml安装Etcd服务。
[wait-control-plane]：等待control-plan部署的Master组件启动。
[apiclient]：检查Master组件服务状态。
[uploadconfig]：更新配置
[kubelet]：使用configMap配置kubelet。
[patchnode]：更新CNI信息到Node上，通过注释的方式记录。
[mark-control-plane]：为当前节点打标签，打了角色Master，和不可调度标签，这样默认就不会使用Master节点来运行Pod。
[bootstrap-token]：生成token记录下来，后边使用kubeadm join往集群中添加节点时会用到
[addons]：安装附加组件CoreDNS和kube-proxy

备注：/etc/kubernetes/manifests/下的文件是以静态POD形式启动，即不通过master创建，所以不通过master管理

4.5为kubectl准备Kubeconfig文件
kubectl默认会在执行的用户家目录下面的.kube目录下寻找config文件。这里是将在初始化时[kubeconfig]步骤生成的admin.conf拷贝到.kube/config。

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

4.6.查看集群状态
[root@openshift1 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-0 Healthy {“health”:“true”}

[root@openshift1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
openshift1 NotReady master 3m21s v1.14.4

4.7复制证书到其他master节点（单master节点略过）

USER=root
CONTROL_PLANE_IPS="192.168.30.129 192.168.30.134"
for host in ${CONTROL_PLANE_IPS}; do
    ssh "${USER}"@$host "mkdir -p /etc/kubernetes/pki/etcd"
    scp /etc/kubernetes/pki/ca.* "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.* "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.* "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/etcd/ca.* "${USER}"@$host:/etc/kubernetes/pki/etcd/
    scp /etc/kubernetes/admin.conf "${USER}"@$host:/etc/kubernetes/
done

4.8在其他master执行,注意–experimental-control-plane参数（单master节点略过此步骤）

kubeadm join 192.168.30.88:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:61ed61986ca4e382544a30d368cd974dfffe1e2e0eb7c831c0d7bd9e1db2ac7a \
    --experimental-control-plane 	

注意：token有效期是有限的，如果旧的token过期，可以使用kubeadm token create --print-join-command重新创建一条token。使用新token

4.9 node节点加入：

[root@openshift3 ~]#  kubeadm join 192.168.30.88:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:61ed61986ca4e382544a30d368cd974dfffe1e2e0eb7c831c0d7bd9e1db2ac7a
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.14" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Activating the kubelet service
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

五、配置集群网络

未配置网络的时候coredns是没有启动成功的

[root@openshift1 ~]# kubectl get pod -n kube-system
NAME                                 READY   STATUS    RESTARTS   AGE
coredns-6bf5777df6-8t7tj             0/1     Pending   0          5m53s
coredns-6bf5777df6-z2sjg             0/1     Pending   0          5m53s
etcd-openshift1                      1/1     Running   0          5m2s
kube-apiserver-openshift1            1/1     Running   0          4m57s
kube-controller-manager-openshift1   1/1     Running   0          5m2s
kube-proxy-bsgx6                     1/1     Running   0          5m53s
kube-scheduler-openshift1            1/1     Running   0          4m51s

配置calico网络
参考：https://docs.projectcalico.org/v3.8/getting-started/kubernetes/installation/calico
官方对两个配置文件说明（所以生产上还是建议使用calico-typha.yaml，注意至少有3个node节点才能启动成功）

• Installing with the Kubernetes API datastore—50 nodes or less
  calico.yaml
• Installing with the Kubernetes API datastore—more than 50 nodes
  calico-typha.yaml

修改Pod默认192.168.0.0/16网段
POD_CIDR="<your-pod-cidr>" \
sed -i -e "s?10.244.0.0/16?$POD_CIDR?g" calico-typha.yaml

生产环境--修改副本数，通常部署3个避免滚动升级出现问题
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: calico-typha
  ...
spec:
  ...
  replicas: <number of replicas>

kubectl apply -f  calico-typha.yaml

测试环境

[root@openshift1 ~]# kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/calico.yaml
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created

kube-proxy开启ip_vs（1.4默认开启）
修改ConfigMap的kube-system/kube-proxy中的config.conf，mode: “ipvs”

*重启kube-proxy
[root@openshift1 ~]# kubectl get pod -n kube-system | grep kube-proxy | awk ‘{system(“kubectl delete pod “$1” -n kube-system”)}’
pod “kube-proxy-bl7nt” deleted
pod “kube-proxy-fsrr5” deleted
pod “kube-proxy-jnf7k” deleted

查看是否开启了ipvs(出现Using ipvs Proxier)

[root@openshift1 ~]# kubectl logs kube-proxy-s9jlv -n kube-system

I0820 09:54:29.434014       1 server_others.go:176] Using ipvs Proxier.
W0820 09:54:29.434398       1 proxier.go:386] IPVS scheduler not specified, use rr by default
I0820 09:54:29.434725       1 server.go:562] Version: v1.14.4
I0820 09:54:29.442667       1 conntrack.go:52] Setting nf_conntrack_max to 131072
I0820 09:54:29.442880       1 config.go:202] Starting service config controller
I0820 09:54:29.442912       1 controller_utils.go:1027] Waiting for caches to sync for service config controller
I0820 09:54:29.443315       1 config.go:102] Starting endpoints config controller
I0820 09:54:29.443340       1 controller_utils.go:1027] Waiting for caches to sync for endpoints config controller
I0820 09:54:29.543141       1 controller_utils.go:1034] Caches are synced for service config controller
I0820 09:54:29.543563       1 controller_utils.go:1034] Caches are synced for endpoints config controller

查看状态

[root@openshift1 ~]# kubectl get node
NAME         STATUS   ROLES    AGE     VERSION
openshift1   Ready    master   19m     v1.14.4
openshift2   Ready    <none>   9m47s   v1.14.4
openshift3   Ready    <none>   8m29s   v1.14.4

查看coredns状态，已经从calico获取ip了（如果没启动成功删掉pod让它重启试试）

[root@openshift1 ~]#  kubectl get pod --all-namespaces -o wide  |grep core
kube-system   coredns-6bf5777df6-8t7tj                  1/1     Running   0          21h   192.168.68.131    openshift3   <none>           <none>
kube-system   coredns-6bf5777df6-z2sjg                  1/1     Running   0          21h   192.168.68.130    openshift3   <none>           <none>

六、部署Dashboard

注：在master节点上进行如下操作
1.创建Dashboard的yaml文件

wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml

sed -i 's/k8s.gcr.io/loveone/g' kubernetes-dashboard.yaml
sed -i '/targetPort:/a\ \ \ \ \ \ nodePort: 30001\n\ \ type: NodePort' kubernetes-dashboard.yaml

kubectl create -f kubernetes-dashboard.yaml

2.查看访问Dashboard的认证令牌

kubectl create serviceaccount  dashboard-admin -n kube-system
kubectl create clusterrolebinding  dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')

3.通过firefox浏览器访问https://192.168.30.133:30001登录即可

七、重置集群（若有需要）

kubeadm reset
停止相关服务
systemctl stop kubelet etcd

八、问题排查

1. tail -f /var/log/messages
2. kubectl describe pod calico-typha-649d9968df-js57r -n kube-system

九、etcd状态检查

进入容器执行命令查看集群状态

kubectl exec -it etcd-openshift1 sh -n kube-system

etcdctl --cert-file /etc/kubernetes/pki/etcd/peer.crt  --key-file /etc/kubernetes/pki/etcd/peer.key --ca-file /etc/kubernetes/pki/etcd/ca.crt --endpoints h
ttps://192.168.30.133:2379 cluster-health
member b6292fdd24cd4e51 is healthy: got healthy result from https://192.168.30.134:2379
member b799094b2a7c90fb is healthy: got healthy result from https://192.168.30.129:2379
member e089a3c3c5e73ae0 is healthy: got healthy result from https://192.168.30.133:2379