1. 环境清理与虚拟机配置

1.1 环境清理

[root@server2 ~]# kubectl delete nodes server4   ##将server3和server4从server2上剥离
[root@server2 ~]# kubectl delete nodes server3 
[root@server4 ~]# kubeadm reset    ##重置server3和server4
[root@server4 ~]# reboot           ##重启server3和server4
[root@server2 ~]# kubectl delete nodes server2    ##删除server2节点并重置重启
[root@server2 ~]# kubeadm  reset 
[root@server2 ~]#  ipvsadm --clear    ##每台机器都需要做
[root@server2 ~]# reboot  
[root@server2 ~]# ipvsadm -l    ##server2、3、4查看ipvs是否存在ipvs策略，如果存在ipvsadm --clear
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

1.2 虚拟机配置

设置两台虚拟机，做高可用。原来的三个做全部做k8s的master。

2. 部署k8s高可用原理图

å¨è¿éæå¥å¾çæè¿°

3. Loadbalancer部署

3.1 配置haproxy

## 1. Loadbalancer部署(负载均衡的部署)(server5作为负载均衡器)
[root@server5 haproxy]# ip addr add  172.25.13.100/24 dev eth0  ##添加vip，为了之后做haproxy高可用
[root@server5 ~]# yum install -y haproxy    ##搭建haproxy
[root@server5 haproxy]# vim haproxy.cfg   #编辑内容如下图

[root@server5 haproxy]# systemctl enable --now haproxy.service   ##启动
[root@server5 haproxy]# cat /var/log/messages      ##出错查看日志
[root@server5 haproxy]# netstat -antlp    ##必须看到6443端口起来
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:6443            0.0.0.0:*               LISTEN      13802/haproxy

4. k8s集群部署

4.1 docker部署

[root@server2 ~]# vim /etc/docker/daemon.json     ##之前都配置过
{
  "registry-mirrors": ["https://reg.westos.org"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
[root@server2 ~]# cat /etc/sysctl.d/docker.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1	

#加载内核模块：（kube_proxy使用IPVS模式）
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4

[root@server2 ~]# yum install ipvsadm -y

4.2 k8s配置(三个master节点)

## 1.修改初始化文件
[root@server2 ~]# kubeadm config print init-defaults > kubeadm-init.yaml  ##生成init文件
[root@server2 ~]# vim kubeadm-init.yaml    ##编辑文件内容如下图
[root@server2 ~]# cat kubeadm-init.yaml 
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 172.25.13.2
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: server2
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "172.25.13.100:6443"
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: reg.westos.org/k8s
#imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.20.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs     ##ipvs模块来做proxy

4.2.1 修改初始化文件

4.2.2 删除server2、3、4所有的镜像

## 2. 删除server2、3、4上面所有的镜像，然后从本地部署k8s需要的镜像
[root@server2 ~]# docker rmi `docker images | grep -v ^REPOSITORY | awk '{print $1":"$2}'`

4.2.3 上传仓库所需镜像

[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/pause:3.2
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/coredns:1.7.0
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/etcd:3.4.13-0
[root@server1 harbor]# docker pull quay.io/coreos/flannel:v0.12.0-amd64
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.20.0
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.20.0
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.20.0
[root@server1 harbor]# docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.20.0

[root@server1 harbor]# docker push reg.westos.org/k8s/kube-apiserver:v1.20.0
[root@server1 harbor]# docker push reg.westos.org/k8s/kube-proxy:v1.20.0
[root@server1 harbor]# docker push reg.westos.org/k8s/kube-scheduler:v1.20.0
[root@server1 harbor]# docker push reg.westos.org/k8s/kube-controller-manager:v1.20.0
[root@server1 harbor]# docker push reg.westos.org/k8s/coredns:1.7.0
[root@server1 harbor]# docker push reg.westos.org/k8s/pause:3.2
[root@server1 harbor]# docker push reg.westos.org/k8s/etcd:3.4.13-0
[root@server1 harbor]# docker push reg.westos.org/k8s/flannel:v0.12.0-amd64

4.2.4 k8s初始化

[root@server2 ~]# kubeadm config images list --config kubeadm-init.yaml   ##后面的版本号是更着init文件中的k8s版本走的，刚才上传的镜像就是现在展示的版本
reg.westos.org/k8s/kube-apiserver:v1.20.0
reg.westos.org/k8s/kube-controller-manager:v1.20.0
reg.westos.org/k8s/kube-scheduler:v1.20.0
reg.westos.org/k8s/kube-proxy:v1.20.0
reg.westos.org/k8s/pause:3.2
reg.westos.org/k8s/etcd:3.4.13-0
reg.westos.org/k8s/coredns:1.7.0

[root@server2 ~]# kubeadm init   --config kubeadm-init.yaml --upload-certs ##初始化，生成两个密钥，一个是加入控制面板，都是master(control-plane),一个是加入集群作为slave。三个都是master，需要共享证书。

wd
[root@server2 ~]# export KUBECONFIG=/etc/kubernetes/admin.conf   ##导入admin认证，不然会出现下面的错误（这是方法一，针对是root用户，可以将上面的命令加入.bashrc环境变量）
Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes")
[root@server2 ~]# vim .bashrc      ##加入环境变量，扩容的master节点也是每一个加一下环境变量
[root@server2 ~]# cat .bashrc | grep KUBE
export KUBECONFIG=/etc/kubernetes/admin.conf
[root@server2 ~]# source .bashrc 

##（方法二，针对不是root用户）
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config


[root@server2 ~]# kubectl get node
NAME      STATUS   ROLES                  AGE   VERSION
server2   Ready    control-plane,master   10m   v1.20.2

[root@server2 ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc  ##添加k8s命令补齐功能

4.2.5 节点扩容

##节点扩容,server3和server4使用刚才生成的加入集群(master)命令进行扩容
##如果加入不进去，可以重新kubeadm reset或者查看/var/lib/etcd是否有文件，里面文件清干净
[root@server2 ~]# kubectl get nodes    ##节点准备好
NAME      STATUS   ROLES                  AGE     VERSION
server2   Ready    control-plane,master   15m     v1.20.2
server3   Ready    control-plane,master   2m41s   v1.20.2
server4   Ready    control-plane,master   119s    v1.20.2

4.2.6 安装网络组件(flannel)

[root@server2 ~]# vim kube-flannel.yml 
[root@server2 ~]# cat kube-flannel.yml   ##这个文件可以网络上下载，修改一下镜像获取路径即可，并修改工作模式从vxlan改为host-gw
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
    - configMap
    - secret
    - emptyDir
    - hostPath
  allowedHostPaths:
    - pathPrefix: "/etc/cni/net.d"
    - pathPrefix: "/etc/kube-flannel"
    - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups: ['extensions']
    resources: ['podsecuritypolicies']
    verbs: ['use']
    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-amd64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: kubernetes.io/arch
                    operator: In
                    values:
                      - amd64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: reg.westos.org/k8s/flannel:v0.12.0-amd64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: reg.westos.org/k8s/flannel:v0.12.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: kubernetes.io/arch
                    operator: In
                    values:
                      - arm64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.12.0-arm64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.12.0-arm64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: kubernetes.io/arch
                    operator: In
                    values:
                      - arm
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.12.0-arm
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.12.0-arm
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-ppc64le
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: kubernetes.io/arch
                    operator: In
                    values:
                      - ppc64le
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.12.0-ppc64le
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.12.0-ppc64le
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-s390x
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: kubernetes.io/arch
                    operator: In
                    values:
                      - s390x
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay.io/coreos/flannel:v0.12.0-s390x
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay.io/coreos/flannel:v0.12.0-s390x
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg

[root@server2 ~]# kubectl apply -f kube-flannel.yml   ##创建flannel
[root@server2 ~]# kubectl -n kube-system get pod | grep coredns | awk '{system("kubectl -n kube-system delete pod "$1"")}'   ##删除pod重新生成（调度会比较慢）
[root@server2 ~]# kubectl -n kube-system delete pod coredns-58796ff66-bmztk --force ##删不了就强只删除

4.3 k8s配置(node节点–需要配置docker和k8s)

4.3.1 前期配置

[root@server2 ~]# cd /etc/yum.repos.d/
[root@server2 yum.repos.d]# ls
CentOS-Base.repo  docker.repo  k8s.repo  redhat.repo  rhel7.6.repo
[root@server2 yum.repos.d]# scp docker.repo k8s.repo server6:/etc/yum.repos.d/

[root@server6 yum.repos.d]# cat k8s.repo 
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0

[root@server6 yum.repos.d]# cat docker.repo   ##修改仓库文件
[docker]
name=docker-ce
baseurl=file:///var/www/html/docker-ce
gpgcheck=0
enabled=1
[root@server6 yum.repos.d]# ll /var/www/html/docker-ce/   ##pub/docs/docker/docker-ce目录
total 463732
-rwxr-xr-x 1 root root 24250280 Mar  6 09:28 containerd.io-1.2.10-3.2.el7.x86_64.rpm
-rwxr-xr-x 1 root root 24416076 Mar  6 09:28 containerd.io-1.2.13-3.1.el7.x86_64.rpm
-rwxr-xr-x 1 root root 26675948 Mar  6 09:28 containerd.io-1.2.13-3.2.el7.x86_64.rpm
-rwxr-xr-x 1 root root 30374084 Mar  6 09:28 containerd.io-1.3.7-3.1.el7.x86_64.rpm
-rwxr-xr-x 1 root root    38144 Mar  6 09:28 container-selinux-2.77-1.el7.noarch.rpm
-rwxr-xr-x 1 root root 22238716 Mar  6 09:28 docker-ce-18.09.9-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 25278520 Mar  6 09:28 docker-ce-19.03.11-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 25285728 Mar  6 09:28 docker-ce-19.03.12-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 25268380 Mar  6 09:28 docker-ce-19.03.13-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 25671976 Mar  6 09:28 docker-ce-19.03.5-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 25697324 Mar  6 09:28 docker-ce-19.03.8-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 16409108 Mar  6 09:28 docker-ce-cli-18.09.9-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 40054796 Mar  6 09:28 docker-ce-cli-19.03.11-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 40044364 Mar  6 09:28 docker-ce-cli-19.03.12-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 40247476 Mar  6 09:28 docker-ce-cli-19.03.13-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 41396672 Mar  6 09:28 docker-ce-cli-19.03.5-3.el7.x86_64.rpm
-rwxr-xr-x 1 root root 41468684 Mar  6 09:28 docker-ce-cli-19.03.8-3.el7.x86_64.rpm
drwxr-xr-x 2 root root     4096 Mar  6 09:28 repodata
[root@server6 yum.repos.d]# yum repolist    ##加载仓库
Loaded plugins: product-id, search-disabled-repos, subscription-manager
This system is not registered with an entitlement server. You can use subscription-manager to register.
repo id                                 repo name                                status
docker                                  docker-ce                                   17
kubernetes                              Kubernetes                                 633
rhel7.6                                 rhel7.6                                  5,152
repolist: 5,802

4.3.2 安装docker并发送所需文件

## 2. 安装并发送所需文件
[root@server6 yum.repos.d]# yum install docker-ce -y
[root@server2 ~]# cd /etc/sysctl.d/
[root@server2 sysctl.d]# scp docker.conf server6:/etc/sysctl.d/  
[root@server2 sysctl.d]# cat docker.conf 
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

[root@server6 ~]# systemctl enable --now docker  ##启动docker

[root@server2 sysctl.d]# cd /etc/docker/
[root@server2 docker]# ls
certs.d  daemon.json  key.json
[root@server2 docker]# scp -r daemon.json certs.d/ server6:/etc/docker/  ##发送配置和证书

4.3.3 做解析

[root@server6 ~]# vim /etc/hosts
[root@server6 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.13.1 server1 reg.westos.org   ##解析
172.25.13.2 server2
172.25.13.3 server3
172.25.13.4 server4
172.25.13.5 server5
172.25.13.6 server6
172.25.13.7 server7
172.25.13.250 westos.westos.org

[root@server6 docker]# systemctl restart docker
[root@server6 docker]# docker info

4.3.4 安装ipvsadm模块

[root@server6 ~]# yum install -y ipvsadm   ##安装ipvsadm模块(需要ipvs模块)
[root@server6 ~]# modprobe ip_vs   ##添加ip_vs模块
[root@server6 ~]# modprobe ip_vs_rr
[root@server6 ~]# modprobe ip_vs_sh
[root@server6 ~]# modprobe ip_vs_wrr
[root@server6 ~]# lsmod | grep ip_vs

4.3.5 节点配置kubeadm，kubectl,kubelet

[root@server6 ~]# yum install kubeadm-1.20.2-0.x86_64 kubelet-1.20.2-0.x86_64 -y  ##安装k8s组件

##禁用swap分区
[root@server6 ~]# swapoff -a 
[root@server6 ~]# vim /etc/fstab 
[root@server6 ~]# cat /etc/fstab 

#
# /etc/fstab
# Created by anaconda on Tue Feb  2 09:03:35 2021
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/rhel-root   /                       xfs     defaults        0 0
UUID=dc5cf3ea-2aca-4a0d-90b1-791ffa119129 /boot                   xfs     defaults        0 0
#/dev/mapper/rhel-swap   swap                    swap    defaults        0 0

##启动kubelet服务
[root@server6 ~]# systemctl enable --now kubelet.service

4.3.6 节点扩容

[root@server6 ~]# kubeadm join 172.25.13.100:6443 --token abcdef.0123456789abcdef \
>     --discovery-token-ca-cert-hash sha256:073e8e7a21f50e2e6bcc46079d7253c22c36f87a9375a4398367c89c39b3cfac
##加入节点命令

[root@server3 ~]# kubectl get nodes   ##master上查看是否添加节点成功
NAME      STATUS   ROLES                  AGE    VERSION
server2   Ready    control-plane,master   105m   v1.20.2
server3   Ready    control-plane,master   102m   v1.20.2
server4   Ready    control-plane,master   100m   v1.20.2
server6   Ready    <none>                 15s    v1.20.2

4.4 测试

[root@server2 ~]# kubectl run demo --image=myapp:v1
pod/demo created
[root@server2 ~]# kubectl get pod
NAME   READY   STATUS    RESTARTS   AGE
demo   1/1     Running   0          4s
[root@server2 ~]# kubectl get pod -o wide 
NAME   READY   STATUS    RESTARTS   AGE   IP           NODE      NOMINATED NODE   READINESS GATES
demo   1/1     Running   0          8s    10.244.3.2   server6   <none>           <none>
[root@server2 ~]# curl 10.244.3.2
Hello MyApp | Version: v1 | <a href="hostname.html">Pod Name</a>

5. 使用pacemaker搭建k8s的高可用(haproxy的高可用)

5.1 安装haproxy

## 1. 安装haproxy
[root@server8 ~]# yum install haproxy -y    ##新开虚拟机安装haproxy
[root@server5 haproxy]# scp haproxy.cfg  server8:/etc/haproxy/   ##将之前server5配置的haproxy文件发送，并作出相应修改（一定要做解析）

5.2 安装pacemaker

## 1. 免密
[root@server5 ~]# ssh-keygen      ##做免密
[root@server5 ~]# ssh-copy-id server8   ##做免密

## 2.安装pacemaker
[root@server5 yum.repos.d]# vim rhel7.6.repo 
[root@server5 yum.repos.d]# cat rhel7.6.repo 
[rhel7.6]
name=rhel7.6
baseurl=http://172.25.13.250/rhel7.6
gpgcheck=0

[HighAvailability]   ##高可用
name =HighAvailability
baseurl=http://172.25.13.250/rhel7.6/addons/HighAvailability
gpgcheck=0

[root@server5 ~]# yum install -y pacemaker pcs psmisc policycoreutils-python  ##安装pacemaker
[root@server5 yum.repos.d]# scp rhel7.6.repo server8:/etc/yum.repos.d/  ##拷贝一份仓库文件
[root@server5 yum.repos.d]# ssh server8 yum install -y pacemaker pcs psmisc policycoreutils-python    ##server8也配置同样的pacemaker

## 3.启动pcs服务
[root@server5 ~]# systemctl enable --now pcsd.service 
[root@server5 ~]# ssh server8 systemctl enable --now pcsd.service   ##启动server8 服务

5.3 配置pacemaker

 ## 1. create a hacluster user
 [root@server5 ~]# passwd hacluster     ##创建一个用户(两个节点使用一个)
[root@server5 ~]# echo westos | passwd --stdin hacluster 
[root@server5 ~]# ssh server8 'echo westos | passwd --stdin hacluster'
[root@server5 ~]# cat /etc/shadow         ##查看是否有密码
[root@server8 ~]# cat /etc/shadow
## 2. 对两个用户进行认证
[root@server5 ~]#  pcs cluster auth server5 server8   
Username: hacluster  ##用户是上一条命令生成的
Password: 
server5: Authorized
server8: Authorized

## 3. 组件集群
[root@server5 ~]#  pcs cluster setup --name mycluster server5 server8  ##集群名字mycluster

## 4. 设置开机自启动集群
[root@server5 ~]#  pcs cluster start --all 
server5: Starting Cluster (corosync)...   ##集群心跳
server8: Starting Cluster (corosync)...   ##pacemaker是资源管理器
server5: Starting Cluster (pacemaker)...
server8: Starting Cluster (pacemaker)...
[root@server5 ~]#  pcs cluster enable --all
server5: Cluster Enabled
server8: Cluster Enabled

5.4 校验(设置stonith)

[root@server5 ~]#  corosync-cfgtool -s
Printing ring status.
Local node ID 1
RING ID 0
	id	= 172.25.13.5
	status	= ring 0 active with no faults
[root@server5 ~]# pcs status           ##查看状态
Cluster name: mycluster

WARNINGS:
No stonith devices and stonith-enabled is not false

[root@server5 ~]#  pcs property set stonith-enabled=false   ##设置stonith为False
[root@server5 ~]# pcs status      ##状态没有警告
[root@server5 ~]# crm_verify -LV      ##验证成功

5.5 配置资源

5.5.1 配置vip资源

## 1. 配置vip资源
[root@server5 ~]#  pcs resource standards   ##查看系统级资源
lsb
ocf
service
systemd
[root@server5 ~]# pcs resource create --help   ##查看资源创建的帮助
[root@server5 ~]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=172.25.13.100 cidr_netmask=24 op monitor interval=30s          ##创建vip资源(op表示监控)
[root@server5 ~]# pcs status    ##查看状态，看是否创建成功

5.5.2 配置haproxy服务资源

[root@server5 ~]# pcs resource create haproxy systemd:haproxy op monitor interval=30s  ##添加haproxy服务资源
[root@server5 ~]# pcs status

5.5.3 资源放到一个组

[root@server5 ~]# pcs resource group add  apiserver vip haproxy   #先起vip,后起服务。把资源加到apiserver这个组。apiserver是组名

5.6 测试主从切换(双机热备)

[root@server5 ~]# pcs node standby 
[root@server5 ~]# pcs status
# kubernetes # docker # linux
Logo
K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 |  韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号​前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。​长夜有穷，真水无香。领略过了 K8s“身在江

avatar K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub

avatar K8S/Kubernetes

k8s 火了！

2020，上云之年，产品云端化成为一种趋势。在一线城市，很多公司都已经构建了自己的私有云环境，比如阿里云、网易云、华为云等。而Kubernetes 作为基于容器编排领域的王者，具备扩展...

avatar K8S/Kubernetes