k8s集群二进制安装部署

k8s集群二进制安装部署机器规划修改主机名配置hosts文件关闭防火墙和selinux关闭交换分区时间同步修改内核参数加载ipvs模块配置工作目录搭建etcd集群配置etcd工作目录创建etcd证书机器规划hostnameipk8s节点k8s-master-0110.0.6.128masterk8s-master-0210.0.6.129masterk8s-master-0310.0.6.130m

天下英雄，今日聚集于此

1491人浏览 · 2021-07-26 12:18:39

天下英雄，今日聚集于此 · 2021-07-26 12:18:39 发布

k8s集群二进制安装部署

（以下命令，如无特殊提示，默认在k8s-master-01上执行）

机器规划（CentOS 7.9 2009 内核3.10+）

hostname	ip	k8s节点
k8s-master-01	10.0.6.128	master
k8s-master-02	10.0.6.129	master
k8s-master-03	10.0.6.130	master
k8s-node-01	10.0.6.131	node
k8s-node-02	10.0.6.132	node

修改主机名

k8s-master-01、k8s-master-02、k8s-master-03、k8s-node-01、k8s-node-02

配置hosts文件

#每台机器上执行
cat >> /etc/hosts << EOF
10.0.6.128 k8s-master-01
10.0.6.129 k8s-master-02
10.0.6.130 k8s-master-03
10.0.6.131 k8s-node-01
10.0.6.132 k8s-node-02
EOF

关闭防火墙和selinux

#每台机器上执行
systemctl stop firewalld
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

关闭交换分区

#每台机器上执行
swapoff -a
sed -i 's/^[^#].*swap*/#&/g' /etc/fstab

时间同步

#每台机器上执行
yum install -y chrony
systemctl start chronyd
systemctl enable chronyd
chronyc sources

修改内核参数

#每台机器上执行
cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

加载ipvs模块

#每台机器上执行
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
lsmod | grep ip_vs
lsmod | grep nf_conntrack_ipv4
yum install -y ipvsadm

配置工作目录

mkdir -p /data/work
yum install sshpass wget -y
ssh-keygen -t rsa -b 2048
#000000是机器root用户的密码
sshpass -p 000000 ssh-copy-id -o StrictHostKeyChecking=no root@k8s-master-02
sshpass -p 000000 ssh-copy-id -o StrictHostKeyChecking=no root@k8s-master-03
sshpass -p 000000 ssh-copy-id -o StrictHostKeyChecking=no root@k8s-node-01
sshpass -p 000000 ssh-copy-id -o StrictHostKeyChecking=no root@k8s-node-02

搭建etcd集群

配置etcd工作目录

mkdir -p /var/lib/etcd/default.etcd
mkdir -p /etc/etcd/ssl

创建etcd证书

cd /data/work/
wget https://hub.fastgit.org/cloudflare/cfssl/releases/download/v1.6.0/cfssl-certinfo_1.6.0_linux_amd64
wget https://hub.fastgit.org/cloudflare/cfssl/releases/download/v1.6.0/cfssljson_1.6.0_linux_amd64
wget https://hub.fastgit.org/cloudflare/cfssl/releases/download/v1.6.0/cfssl_1.6.0_linux_amd64

chmod +x cfssl*
mv cfssl_1.6.0_linux_amd64 /usr/local/bin/cfssl
mv cfssljson_1.6.0_linux_amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_1.6.0_linux_amd64 /usr/local/bin/cfssl-certinfo

cat > ca-csr.json <<CB
{
  "CN": "kubernetes",
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hubei",
      "L": "Wuhan",
      "O": "k8s",
      "OU": "system"
    }
  ],
  "ca": {
          "expiry": "87600h"
  }
}
CB

cfssl gencert -initca ca-csr.json  | cfssljson -bare ca

cat > ca-config.json <<CB
{
  "signing": {
      "default": {
          "expiry": "87600h"
        },
      "profiles": {
          "kubernetes": {
              "usages": [
                  "signing",
                  "key encipherment",
                  "server auth",
                  "client auth"
              ],
              "expiry": "87600h"
          }
      }
  }
}
CB

cat > etcd-csr.json <<CB
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "10.0.6.128",
    "10.0.6.129",
    "10.0.6.130"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "CN",
    "ST": "Hubei",
    "L": "Wuhan",
    "O": "k8s",
    "OU": "system"
  }]
}
CB

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson  -bare etcd
ls etcd*.pem

注：
CN：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)；浏览器使用该字段验证网站是否合法；
O：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)

部署etcd集群

wget https://hub.fastgit.org/etcd-io/etcd/releases/download/v3.4.13/etcd-v3.4.13-linux-amd64.tar.gz
tar -xf etcd-v3.4.13-linux-amd64.tar.gz
cp -p etcd-v3.4.13-linux-amd64/etcd* /usr/local/bin/
scp etcd-v3.4.13-linux-amd64/etcd* k8s-master-02:/usr/local/bin/
scp etcd-v3.4.13-linux-amd64/etcd* k8s-master-03:/usr/local/bin/

cat > etcd.conf <<EOF
#[Member]
ETCD_NAME="etcd1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.0.6.128:2380"
ETCD_LISTEN_CLIENT_URLS="https://10.0.6.128:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.6.128:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://10.0.6.128:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://10.0.6.128:2380,etcd2=https://10.0.6.129:2380,etcd3=https://10.0.6.130:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

cat > etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=-/etc/etcd/etcd.conf
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd \
  --cert-file=/etc/etcd/ssl/etcd.pem \
  --key-file=/etc/etcd/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/etcd/ssl/ca.pem \
  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \
  --peer-client-cert-auth \
  --client-cert-auth
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cp ca*.pem /etc/etcd/ssl/
cp etcd*.pem /etc/etcd/ssl/
cp etcd.conf /etc/etcd/
cp etcd.service /usr/lib/systemd/system/
for i in k8s-master-02 k8s-master-03
do 
	ssh -o StrictHostKeyChecking=no root@$i "mkdir -p /var/lib/etcd/default.etcd && mkdir -p /etc/etcd/ssl"
	scp etcd.conf $i:/etc/etcd/
  scp etcd*.pem ca*.pem $i:/etc/etcd/ssl/
  scp etcd.service $i:/usr/lib/systemd/system/
done

注：
执行之后分别在机器k8s-master-02和k8s-master-02上修改/etc/etcd/etcd.conf配置文件中etcd名字和ip

启动etcd集群

systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd.service
systemctl status etcd
for i in k8s-master-02 k8s-master-03
do
	ssh -o StrictHostKeyChecking=no root@$i "systemctl daemon-reload && systemctl enable etcd.service && systemctl start etcd.service && systemctl status etcd"
done

注：
第一次启动可能会卡一段时间，因为节点会等待其他节点启动

查看集群状态

ETCDCTL_API=3
/usr/local/bin/etcdctl --write-out=table --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://10.0.6.128:2379,https://10.0.6.129:2379,https://10.0.6.130:2379 endpoint health

kubernetes组件部署

(操作在 k8s-master-01 上进行)

下载安装包

wget https://dl.k8s.io/v1.20.1/kubernetes-server-linux-amd64.tar.gz
tar -xf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-apiserver kube-controller-manager kube-scheduler kubectl /usr/local/bin/
for i in k8s-master-02 k8s-master-03
do
	scp kube-apiserver kube-controller-manager kube-scheduler kubectl $i:/usr/local/bin/
done
for i in k8s-node-01 k8s-node-02;do scp kubelet kube-proxy $i:/usr/local/bin/;done

创建工作目录

mkdir -p /etc/kubernetes/
mkdir -p /etc/kubernetes/ssl
mkdir -p /var/log/kubernetes

部署api-server

创建csr请求文件

cd /data/work/
cat > kube-apiserver-csr.json <<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "10.0.6.128",
    "10.0.6.129",
    "10.0.6.130",
    "10.0.6.200",
    "10.255.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hubei",
      "L": "Wuhan",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF

注：
hosts 字段需要指定授权使用该证书的 IP或域名列表。
由于该证书后续被 kubernetes master 集群使用，需要将master节点的IP都填上，同时还需要填写 service 网络的首个IP。(一般是 kube-apiserver 指定的 service-cluster-ip-range 网段的第一个IP，如 10.255.0.1),还需要将VIP也填上，VIP我们采用haproxy+keepalive，在三个master节点上搭建虚拟IP。

生成证书和token文件

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

创建配置文件

cat > kube-apiserver.conf <<EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
  --anonymous-auth=false \
  --bind-address=10.0.6.128 \
  --secure-port=6443 \
  --advertise-address=10.0.6.128 \
  --insecure-port=0 \
  --authorization-mode=Node,RBAC \
  --runtime-config=api/all=true \
  --enable-bootstrap-token-auth \
  --service-cluster-ip-range=10.255.0.0/16 \
  --token-auth-file=/etc/kubernetes/token.csv \
  --service-node-port-range=30000-50000 \
  --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem  \
  --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
  --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
	--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  \
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \
  --etcd-cafile=/etc/etcd/ssl/ca.pem \
  --etcd-certfile=/etc/etcd/ssl/etcd.pem \
  --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
  --etcd-servers=https://10.0.6.128:2379,https://10.0.6.129:2379,https://10.0.6.130:2379 \
  --enable-swagger-ui=true \
  --allow-privileged=true \
  --apiserver-count=3 \
  --audit-log-maxage=30 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-path=/var/log/kube-apiserver-audit.log \
  --event-ttl=1h \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=4"
EOF

注：
–logtostderr：启用日志
–v：日志等级
–log-dir：日志目录
–etcd-servers：etcd集群地址
–bind-address：监听地址
–secure-port：https安全端口
–advertise-address：集群通告地址
–allow-privileged：启用授权
–service-cluster-ip-range：Service虚拟IP地址段
–enable-admission-plugins：准入控制模块
–authorization-mode：认证授权，启用RBAC授权和节点自管理
–enable-bootstrap-token-auth：启用TLS bootstrap机制
–token-auth-file：bootstrap token文件
–service-node-port-range：Service nodeport类型默认分配端口范围
–kubelet-client-xxx：apiserver访问kubelet客户端证书
–tls-xxx-file：apiserver https证书
–etcd-xxxfile：连接Etcd集群证书
–audit-log-xxx：审计日志

创建服务启动文件

cat > kube-apiserver.service <<'EOF'
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

同步相关文件到各个节点

\cp ca*.pem /etc/kubernetes/ssl/
\cp kube-apiserver*.pem /etc/kubernetes/ssl/
\cp token.csv /etc/kubernetes/
\cp kube-apiserver.conf /etc/kubernetes/
\cp kube-apiserver.service /usr/lib/systemd/system/

for i in k8s-master-02 k8s-master-03
do
	ssh -o StrictHostKeyChecking=no root@$i "mkdir -p /etc/kubernetes/ssl && mkdir -p /var/log/kubernetes"
  scp token.csv kube-apiserver.conf $i:/etc/kubernetes/
  scp kube-apiserver*.pem ca*.pem $i:/etc/kubernetes/ssl/
  scp kube-apiserver.service $i:/usr/lib/systemd/system/
done

注：执行之后分别在机器k8s-master-02和k8s-master-02上修改/etc/kubernetes/kube-apiserver.conf配置文件中IP地址为本机器IP

启动kube-apiserver服务

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

for i in k8s-master-02 k8s-master-03
do
	ssh -o StrictHostKeyChecking=no root@$i "systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver && systemctl status kube-apiserver"
done

set -x
curl --insecure https://k8s-master-01:6443/
curl --insecure https://k8s-master-02:6443/
curl --insecure https://k8s-master-03:6443/
set +x

搭建api-server负载均衡

yum install haproxy keepalived -y && systemctl enable haproxy keepalived
cat << EOF > haproxy.cfg 
global
    log /dev/log    local0
    log /dev/log    local1 notice
    chroot /var/lib/haproxy
    stats socket /var/run/haproxy-admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon
    nbproc 1
defaults
    log     global
    timeout connect 5000
    timeout client  10m
    timeout server  10m
listen  admin_stats
    bind 0.0.0.0:10080
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login\ Haproxy
    stats auth admin:DreamCatcher
    stats hide-version
    stats admin if TRUE
listen kube-master
    bind 0.0.0.0:10443
    mode tcp
    option tcplog
    balance source
    server k8s-master-01 k8s-master-01:6443 check inter 2000 fall 2 rise 2 weight 1
    server k8s-master-02 k8s-master-02:6443 check inter 2000 fall 2 rise 2 weight 1
    server k8s-master-03 k8s-master-03:6443 check inter 2000 fall 2 rise 2 weight 1
EOF

for i in k8s-master-02 k8s-master-03
do
	ssh -o StrictHostKeyChecking=no root@$i "yum install haproxy keepalived -y && systemctl enable haproxy keepalived"
done

weight=103

interfaceName=$(ip a | grep -i 10.0.6 -B 2 | awk 'NR==1{print $2}' | sed 's/://')

cat << EOF > keepalived.conf
global_defs {
    router_id k8s-master-01
}
    
vrrp_script check-haproxy {
    script "sudo killall -0 haproxy"
    interval 5
    weight -30
}
    
vrrp_instance VI-kube-master {
    state MASTER
    priority $weight
    dont_track_primary
    interface $interfaceName
    virtual_router_id 68
    advert_int 3
    track_script {
        check-haproxy
    }
    virtual_ipaddress {
        10.0.6.200
    }
}
EOF

\cp haproxy.cfg /etc/haproxy/haproxy.cfg
\cp keepalived.conf  /etc/keepalived/keepalived.conf 
systemctl start haproxy keepalived && systemctl enable haproxy keepalived

for i in k8s-master-02 k8s-master-03
do
 ((weight=$weight-1))
	scp haproxy.cfg $i:/etc/haproxy/haproxy.cfg
  sed -ri "s+(\srouter_id ).*$+\1${i}+g" keepalived.conf
  sed -ri "s+(\priority ).*$+\1${weight}+g" keepalived.conf
	scp keepalived.conf $i:/etc/keepalived/
  ssh -o StrictHostKeyChecking=no root@$i "systemctl start haproxy keepalived && systemctl enable haproxy keepalived"
done

注：
k8s-master-01是主机名称，其他Master节点替换成对应主机的名称
10.0.6.200是虚拟IP，Node节点可以通过该IP进行访问Master
interfaceName 是获取到的网卡名称

查看 haproxy 状态页面

curl 10.0.6.200:10443/status

部署kubectl

创建csr请求文件

cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hubei",
      "L": "Wuhan",
      "O": "system:masters",             
      "OU": "system"
    }
  ]
}
EOF

说明：
后续 kube-apiserver 使用 RBAC 对客户端(如 kubelet、kube-proxy、Pod)请求进行授权；
kube-apiserver 预定义了一些 RBAC 使用的 RoleBindings，如 cluster-admin 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予了调用kube-apiserver 的所有 API的权限；
O指定该证书的 Group 为 system:masters，kubelet 使用该证书访问 kube-apiserver 时，由于证书被 CA 签名，所以认证通过，同时由于证书用户组为经过预授权的 system:masters，所以被授予访问所有 API 的权限；
注：
这个admin 证书，是将来生成管理员用的kube config 配置文件用的，现在我们一般建议使用RBAC 来对kubernetes 进行角色权限控制， kubernetes 将证书中的CN 字段作为User， O 字段作为 Group；
“O”: “system:masters”, 必须是system:masters，否则后面kubectl create clusterrolebinding报错。

生成证书

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
cp admin*.pem /etc/kubernetes/ssl/

创建kubeconfig配置文件

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://10.0.6.200:10443 --kubeconfig=kube.config
kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config
kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config
kubectl config use-context kubernetes --kubeconfig=kube.config
mkdir ~/.kube
cp kube.config ~/.kube/config
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

注：
kubeconfig 为 kubectl 的配置文件，包含访问 apiserver 的所有信息，如 apiserver 地址、CA 证书和自身使用的证书

查看集群组件状态

kubectl cluster-info
kubectl get componentstatuses
kubectl get all --all-namespaces

在这里插入图片描述

同步kubectl配置文件到其他节点

for i in k8s-master-02 k8s-master-03
do
  ssh -o StrictHostKeyChecking=no root@$i "mkdir /root/.kube/"
  scp /root/.kube/config $i:/root/.kube/
done

配置kubectl子命令补全

所有机器上操作，可不做

yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
kubectl completion bash > ~/.kube/completion.bash.inc
source '/root/.kube/completion.bash.inc'  
source ~/.bash_profile

部署kube-controller-manager

创建csr请求文件

cat > kube-controller-manager-csr.json <<EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "10.0.6.128",
      "10.0.6.129",
      "10.0.6.130"
    ],
    "names": [
      {
        "C": "CN",
        "ST": "Hubei",
        "L": "Wuhan",
        "O": "system:kube-controller-manager",
        "OU": "system"
      }
    ]
}
EOF

注：
hosts 列表包含所有 kube-controller-manager 节点 IP；
CN 为 system:kube-controller-manager、O 为 system:kube-controller-manager，kubernetes 内置的 ClusterRoleBindings system:kube-controller-manager 赋予 kube-controller-manager 工作所需的权限

生成证书

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
ls kube-controller-manager*.pem

创建kube-controller-manager的kubeconfig

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://10.0.6.200:10443 --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig

创建配置文件

cat > kube-controller-manager.conf <<EOF
KUBE_CONTROLLER_MANAGER_OPTS="--port=0 \
  --secure-port=10252 \
  --bind-address=127.0.0.1 \
  --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
  --service-cluster-ip-range=10.255.0.0/16 \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
  --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --allocate-node-cidrs=true \
  --cluster-cidr=10.0.0.0/16 \
  --experimental-cluster-signing-duration=87600h \
  --root-ca-file=/etc/kubernetes/ssl/ca.pem \
  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --leader-elect=true \
  --feature-gates=RotateKubeletServerCertificate=true \
  --controllers=*,bootstrapsigner,tokencleaner \
  --horizontal-pod-autoscaler-use-rest-clients=true \
  --horizontal-pod-autoscaler-sync-period=10s \
  --tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
  --use-service-account-credentials=true \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2"
EOF

创建启动文件

cat > kube-controller-manager.service <<'EOF'
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

同步相关文件到各个节点

\cp kube-controller-manager*.pem /etc/kubernetes/ssl/
\cp kube-controller-manager.kubeconfig kube-controller-manager.conf /etc/kubernetes/
\cp kube-controller-manager.service /usr/lib/systemd/system/

scp kube-controller-manager*.pem k8s-master-02:/etc/kubernetes/ssl/
scp kube-controller-manager*.pem k8s-master-03:/etc/kubernetes/ssl/
scp kube-controller-manager.kubeconfig kube-controller-manager.conf k8s-master-02:/etc/kubernetes/
scp kube-controller-manager.kubeconfig kube-controller-manager.conf k8s-master-03:/etc/kubernetes/
scp kube-controller-manager.service k8s-master-02:/usr/lib/systemd/system/
scp kube-controller-manager.service k8s-master-03:/usr/lib/systemd/system/

启动服务

systemctl daemon-reload 
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager

for i in k8s-master-02 k8s-master-03
do
  ssh -o StrictHostKeyChecking=no root@$i "systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl start kube-controller-manager && systemctl status kube-controller-manager"
done

部署kube-scheduler

创建csr请求文件

cat > kube-scheduler-csr.json <<EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "10.0.6.128",
      "10.0.6.129",
      "10.0.6.130"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "Hubei",
        "L": "Wuhan",
        "O": "system:kube-scheduler",
        "OU": "system"
      }
    ]
}
EOF

注：
hosts 列表包含所有 kube-scheduler 节点 IP；
CN 为 system:kube-scheduler、O 为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限。

生成证书

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
ls kube-scheduler*.pem

创建kube-scheduler的kubeconfig

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://10.0.6.200:10443 --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig

创建配置文件

cat > kube-scheduler.conf <<EOF
KUBE_SCHEDULER_OPTS="--address=127.0.0.1 \
--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
--leader-elect=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2"
EOF

创建服务启动文件

cat > kube-scheduler.service <<'EOF'
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

同步相关文件到各个节点

\cp kube-scheduler*.pem /etc/kubernetes/ssl/
\cp kube-scheduler.kubeconfig /etc/kubernetes/
\cp kube-scheduler.conf /etc/kubernetes/
\cp kube-scheduler.service /usr/lib/systemd/system/
scp kube-scheduler*.pem k8s-master-02:/etc/kubernetes/ssl/
scp kube-scheduler*.pem k8s-master-03:/etc/kubernetes/ssl/
scp kube-scheduler.kubeconfig kube-scheduler.conf k8s-master-02:/etc/kubernetes/
scp kube-scheduler.kubeconfig kube-scheduler.conf k8s-master-03:/etc/kubernetes/
scp kube-scheduler.service k8s-master-02:/usr/lib/systemd/system/
scp kube-scheduler.service k8s-master-03:/usr/lib/systemd/system/

启动服务

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler

for i in k8s-master-02 k8s-master-03
do
  ssh -o StrictHostKeyChecking=no root@$i "systemctl daemon-reload && systemctl enable kube-scheduler && systemctl start kube-scheduler && systemctl status kube-scheduler"
done

部署docker

安装docker

（每台机器上执行）

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum install -y docker-ce
systemctl enable docker
systemctl start docker
docker --version

修改docker源和驱动

cat > /etc/docker/daemon.json << EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": [
        "https://1nj0zren.mirror.aliyuncs.com",
        "https://kfwkfulq.mirror.aliyuncs.com",
        "https://2lqq34jg.mirror.aliyuncs.com",
        "https://pee6w651.mirror.aliyuncs.com",
        "http://hub-mirror.c.163.com",
        "https://docker.mirrors.ustc.edu.cn",
        "http://f1361db2.m.daocloud.io",
        "https://registry.docker-cn.com"
    ]
}
EOF

systemctl restart docker
docker info | grep "Cgroup Driver"

for i in k8s-master-02 k8s-master-03 k8s-node-01 k8s-node-02
do
  scp /etc/docker/daemon.json $i:/etc/docker/
  ssh -o StrictHostKeyChecking=no root@$i "systemctl restart docker && docker info | grep 'Cgroup Driver'"
done

下载依赖镜像

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2 k8s.gcr.io/pause:3.2
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0
docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0 k8s.gcr.io/coredns:1.7.0
docker rmi registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0

部署kubelet

创建kubelet-bootstrap的kubeconfig

BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://10.0.6.200:10443 --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

创建配置文件

cat > kubelet.json <<EOF
{
  "kind": "KubeletConfiguration",
  "apiVersion": "kubelet.config.k8s.io/v1beta1",
  "authentication": {
    "x509": {
      "clientCAFile": "/etc/kubernetes/ssl/ca.pem"
    },
    "webhook": {
      "enabled": true,
      "cacheTTL": "2m0s"
    },
    "anonymous": {
      "enabled": false
    }
  },
  "authorization": {
    "mode": "Webhook",
    "webhook": {
      "cacheAuthorizedTTL": "5m0s",
      "cacheUnauthorizedTTL": "30s"
    }
  },
  "address": "10.0.6.131",
  "port": 10250,
  "readOnlyPort": 10255,
  "cgroupDriver": "systemd",
  "hairpinMode": "promiscuous-bridge",
  "serializeImagePulls": false,
  "featureGates": {
    "RotateKubeletClientCertificate": true,
    "RotateKubeletServerCertificate": true
  },
  "clusterDomain": "cluster.local.",
  "clusterDNS": ["10.255.0.2"]
}
EOF

注：
如果docker的驱动为systemd，cgroupDriver处修改为systemd。如果是cgroupfs，修改为cgroupfs此处设置很重要，否则后面node节点无法加入到集群。

创建启动文件

cat > kubelet.service <<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \
  --cert-dir=/etc/kubernetes/ssl \
  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
  --config=/etc/kubernetes/kubelet.json \
  --network-plugin=cni \
  --pod-infra-container-image=k8s.gcr.io/pause:3.2 \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

注：
–network-plugin：启用CNI
–kubeconfig：空路径，会自动生成，后面用于连接apiserver
–bootstrap-kubeconfig：首次启动向apiserver申请证书
–config：配置参数文件
–cert-dir：kubelet证书生成目录
–pod-infra-container-image：管理Pod网络容器的镜像

同步相关文件到各个node节点

for i in k8s-node-01 k8s-node-02
do
  ssh -o StrictHostKeyChecking=no root@$i "mkdir -p /etc/kubernetes/ssl"
  scp kubelet-bootstrap.kubeconfig kubelet.json $i:/etc/kubernetes/
  scp ca.pem $i:/etc/kubernetes/ssl/
  scp kubelet.service $i:/usr/lib/systemd/system/
done

注：执行完成后，到每个node节点上修改/etc/kubernetes/kubelet.json配置文件address改为各个节点的ip地址

启动服务

for i in k8s-node-01 k8s-node-02
do
  ssh -o StrictHostKeyChecking=no root@$i "mkdir /var/lib/kubelet && mkdir -p /var/log/kubernetes && systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet && systemctl status kubelet"
done

确认kubelet服务启动成功后，接着到master上Approve一下bootstrap请求。执行如下命令可以看到两个worker节点分别发送了一个 CSR 请求：

kubectl get csr

在这里插入图片描述

kubectl certificate approve node-csr-Kxw_9iHkEr1G93Q4h-tmu0YkUKEjQDnQqdR64s7En5g
kubectl certificate approve node-csr-ripnh6UJT9kv8-MbJ0DY0hVetly3aO6k1vaR7Bwmc5I

kubectl get csr
kubectl get nodes

在这里插入图片描述

部署kube-proxy

创建csr请求文件

cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Hubei",
      "L": "Wuhan",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF

生成证书

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
ls kube-proxy*.pem

创建kubeconfig文件

kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://10.0.6.200:10443 --kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

创建kube-proxy配置文件

cat > kube-proxy.yaml <<EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 10.0.6.131
clientConnection:
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
clusterCIDR: 192.168.0.0/16
healthzBindAddress: 10.0.6.131:10256
kind: KubeProxyConfiguration
metricsBindAddress: 10.0.6.131:10249
mode: "ipvs"
EOF

注：clusterCIDR此处网段必须与网络组件网段保持一致，否则部署网络组件时会报错

创建服务启动文件

cat > kube-proxy.service <<EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/kube-proxy.yaml \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

同步文件到各个节点


for i in k8s-node-01 k8s-node-02
do
	scp kube-proxy.kubeconfig kube-proxy.yaml $i:/etc/kubernetes/
  scp kube-proxy.service $i:/usr/lib/systemd/system/
done

注：执行完成后，到个node节点修改配置文件/etc/kubernetes/kube-proxy.yaml中address为各节点的实际IP

启动服务

for i in k8s-node-01 k8s-node-02
do
  ssh -o StrictHostKeyChecking=no root@$i "mkdir -p /var/lib/kube-proxy && systemctl daemon-reload && systemctl enable kube-proxy && systemctl start kube-proxy && systemctl status kube-proxy"
done

配置网络组件

wget https://docs.projectcalico.org/v3.14/manifests/calico.yaml
kubectl apply -f calico.yaml

过一会儿再来查看各个节点，均为Ready状态

kubectl get pods -A
kubectl get nodes

在这里插入图片描述

部署coredns

wget https://raw.fastgit.org/coredns/deployment/master/kubernetes/coredns.yaml.sed
mv coredns.yaml.sed coredns.yaml
#此处需要对coredns.yaml文件做一些修改，再apply
kubectl apply -f coredns.yaml

修改内容对比如下：
在这里插入图片描述
注：clusterIP应该与kubelet配置文件中的clusterDNS相同

验证

部署nginx

cat > nginx.yaml <<EOF
---
apiVersion: v1
kind: ReplicationController
metadata:
  name: nginx-controller
spec:
  replicas: 2
  selector:
    name: nginx
  template:
    metadata:
      labels:
        name: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.19.6
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service-nodeport
spec:
  ports:
    - port: 80
      targetPort: 80
      nodePort: 30001
      protocol: TCP
  type: NodePort
  selector:
    name: nginx
EOF

kubectl apply -f nginx.yaml
kubectl get svc
kubectl get pods -A

在这里插入图片描述

验证nginx

kubectl exec -it nginx-controller-gzx7q -- /bin/bash
curl nginx-service-nodeport

在这里插入图片描述
访问nginx：

K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 | 韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。长夜有穷，真水无香。领略过了 K8s“身在江

K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub