2.二进制部署K8s集群
1网段规划主机节点网段192.168.200.0/24Service网段10.244.0.0/16Pod网段10.96.0.0/16网段不能有冲突2集群资源配置Master节点4C8G * 3Node节点4C8G * 33系统设置 – 所有节点关闭Selinux# 临时关闭setenforce 0# 永久关闭sed -i "s/SELINUX=enforcing/SELINUX=disabled/
·
1 网段规划
主机节点网段
192.168.200.0/24
Service网段
10.244.0.0/16
Pod网段
10.96.0.0/16
网段不能有冲突
2 集群资源配置
Master节点
4C8G * 3
Node节点
4C8G * 3
3 系统设置 – 所有节点
关闭Selinux
# 临时关闭
setenforce 0
# 永久关闭
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
关闭防火墙
iptables -F
systemctl disable --now firewalld
配置Hosts – 所有节点
192.168.31.101 k8s-master-01
192.168.31.102 k8s-master-02
192.168.31.103 k8s-master-03
192.168.31.104 k8s-node-01
192.168.31.105 k8s-node-02
192.168.31.106 k8s-node-03
192.168.31.100 k8s-master-lb # 如果不是高可用集群，该IP为Master01的IP
配置YUM源
# 阿里云yum源
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
# docker依赖工具
yum install -y yum-utils device-mapper-persistent-data lvm2
# 安装docker的yum源
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 配置k8s的yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 修改阿里云yum源的域名
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
# 安装必备工具
yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y
关闭swap分区
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
配置时间同步
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
配置Limit
ulimit -SHn 65535
cat >> /etc/security/limits.conf<< EOF
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF
Master01节点到其他节点免密
ssh-keygen -t rsa
for i in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02 k8s-node03;do ssh-copy-id -i .ssh/id_rsa.pub $i;done
下载所有会用到的配置文件
下载安装所有的源码文件：
mkdir /data/src
cd /data/src ; git clone https://github.com/dotbalo/k8s-ha-install.git
如果无法下载就下载：https://gitee.com/dukuan/k8s-ha-install.git
[root@temp k8s-ha-install]# git branch -a
* master
remotes/origin/HEAD -> origin/master
remotes/origin/manual-installation
remotes/origin/manual-installation-v1.16.x
remotes/origin/manual-installation-v1.17.x
remotes/origin/manual-installation-v1.18.x
remotes/origin/manual-installation-v1.19.x
remotes/origin/manual-installation-v1.20.x
remotes/origin/manual-installation-v1.20.x-csi-hostpath
remotes/origin/manual-installation-v1.21.x
remotes/origin/manual-installation-v1.22.x
remotes/origin/manual-installation-v1.23.x
remotes/origin/manual-installation-v1.24.x
remotes/origin/master
[root@temp k8s-ha-install]# git checkout manual-installation-v1.23.x
升级系统
yum update -y --exclude=kernel*
升级内核到4.19
mkdir -p /data/src
cd /data/src
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm
wget http://193.49.22.109/elrepo/kernel/el7/x86_64/RPMS/kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm
# 从master01节点传到其他节点：
for i in k8s-master02 k8s-master03 k8s-node01 k8s-node02 k8s-node03;do scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm $i:/data/src/; done
# 所有节点安装内核
cd /data/src/ && yum localinstall -y kernel-ml*
# 更新启动内核
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
# 检查默认内核是不是4.19
grubby --default-kernel
# 重启系统
# 检查重启后系统内核是不是4.19
uname -a
配置IPVS
# 安装ipvsadm
yum install ipvsadm ipset sysstat conntrack libseccomp -y
# 加载ipvs模块
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
# 修改ipvs配置
cat >>/etc/modules-load.d/ipvs.conf<<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
# 加载ipvs配置
systemctl enable --now systemd-modules-load.service
配置内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
net.ipv4.conf.all.route_localnet = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system
重启服务器
配置完内核后，重启服务器，保证重启后内核依旧加载
reboot
lsmod | grep --color=auto -e ip_vs -e nf_conntrack
4 K8s组件和Runtime安装
安装Containerd作为Runtime – 所有节点
# 所有节点安装docker-ce-20.10：
yum install docker-ce-20.10.* docker-ce-cli-20.10.* -y
-- 无需启动Docker，只需要配置和启动Containerd即可。
配置Containerd所需的模块
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
# 加载模块
modprobe -- overlay
modprobe -- br_netfilter
配置内核参数
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system
修改Containerd配置文件
# 创建配置目录
mkdir -p /etc/containerd
# 生成默认配置文件
containerd config default | tee /etc/containerd/config.toml
- 将Containerd的Cgroup改为Systemd
vim /etc/containerd/config.toml
- 找到containerd.runtimes.runc.options，添加SystemdCgroup = true
- 将sandbox_image的Pause镜像改成符合自己版本的地址
# 使用Containerd必须在这里修改
registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6
# 如图
启动Containerd,并配置开机自启动
systemctl daemon-reload
systemctl enable --now containerd
所有节点配置crictl客户端连接的运行时位置
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
- 配置完成测试ctr命令是否可用
5 安装高可用组件
安装HAProxy和KeepAlived
# 所有Master节点
yum install keepalived haproxy -y
修改Haproxy配置, 所有Master节点配置相同
- apiserver 6443端口映射VIP16443端口
cat > /etc/haproxy/haproxy.cfg << EOF
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend monitor-in
bind *:33305
mode http
option httplog
monitor-uri /monitor
frontend k8s-master
bind 0.0.0.0:16443
bind 127.0.0.1:16443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-master01 192.168.31.101:6443 check
server k8s-master02 192.168.31.102:6443 check
server k8s-master03 192.168.31.103:6443 check
EOF
配置KeepAlived
注意每个节点的IP和网卡配置不一样 ,需要修改
- Master-01
cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
script_user root
enable_script_security
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
mcast_src_ip 192.168.31.101
virtual_router_id 51
priority 101
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.31.100
}
track_script {
chk_apiserver
}
}
EOF
- Master-02
cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
script_user root
enable_script_security
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
mcast_src_ip 192.168.31.102
virtual_router_id 51
priority 101
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.31.100
}
track_script {
chk_apiserver
}
}
EOF
- Master-03
cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
script_user root
enable_script_security
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
mcast_src_ip 192.168.31.103
virtual_router_id 51
priority 101
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.31.100
}
track_script {
chk_apiserver
}
}
EOF
- 所有master节点配置KeepAlived健康检查文件
cat >>/etc/keepalived/check_apiserver.sh<<EOF
#!/bin/bash
err=0
for k in $(seq 1 3)
do
check_code=$(pgrep haproxy)
if [[ $check_code == "" ]]; then
err=$(expr $err + 1)
sleep 1
continue
else
err=0
break
fi
done
if [[ $err != "0" ]]; then
echo "systemctl stop keepalived"
/usr/bin/systemctl stop keepalived
exit 1
else
exit 0
fi
EOF
# 加执行权限
chmod +x /etc/keepalived/check_apiserver.sh
配置haproxy和keepalived开机自启动,
systemctl daemon-reload
systemctl enable --now haproxy keepalived
检查Haproxy和keepalived的状态
ping 192.168.31.100
telnet 192.168.31.100 16443
# 如果ping不通且telnet没有出现 ] ，则认为VIP不可用,需要排查keepalived的问题
# 比如防火墙和selinux，haproxy和keepalived的状态
# 所有节点查看防火墙状态必须为disable和inactive：systemctl status firewalld
# 所有节点查看selinux状态， getenforce执行结果为Permissive,或者Disabled状态
# master节点查看监听端口：netstat -lntp,
6 K8S及ETCD安装
1.查看需要的版本
# kubernetes
https://github.com/kubernetes/kubernetes/tree/release-1.23/CHANGELOG
# etcd
https://github.com/etcd-io/etcd/releases/
2.下载
# Kubernetes server 二进制包
wget https://dl.k8s.io/v1.23.6/kubernetes-server-linux-amd64.tar.gz
# etcd 二进制包
wget https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz
3.解压kubernetes安装文件
tar -xf kubernetes-server-linux-amd64.tar.gz \
--strip-components=3 -C /usr/local/bin \
kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
4.解压etcd安装文件
tar -zxvf etcd-v3.5.1-linux-amd64.tar.gz \
--strip-components=1 -C /usr/local/bin \
etcd-v3.5.1-linux-amd64/etcd{,ctl}
5.查看版本
[root@k8s-master-01 src]# kubelet --version
Kubernetes v1.23.6
[root@k8s-master-01 src]# etcdctl version
etcdctl version: 3.5.1
API version: 3.5
6.将组件发送到其他节点
MasterNodes='k8s-master-02 k8s-master-03'
WorkNodes='k8s-node-01 k8s-node-02 k8s-node-03'
for NODE in $MasterNodes; do echo $NODE; scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done
for NODE in $WorkNodes; do scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done
7.所有节点创建/usr/local/cni/bin目录
mkdir -p /usr/local/cni/bin
8.切换分支
# Master01节点切换到1.23.x分支
cd /data/src/k8s-ha-install
git checkout manual-installation-v1.23.x
7 生成证书
下载生成证书工具
wget "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64" -O /usr/local/bin/cfssl
wget "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson
ETCD证书
- 所有Master节点创建etcd证书目录
mkdir /etc/etcd/ssl -p
- 所有节点创建kubernetes相关目录
mkdir -p /etc/kubernetes/pki
- Master01节点生成etcd证书
cd /data/src/k8s-ha-install/pki
## 生成etcd CA证书和CA证书的key
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca
# etcd-ca-csr.json: 证书签名请求文件，配置了一些域名、公司、单位等信息
cat etcd-ca-csr.json
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "Etcd Security"
}
],
"ca": {
"expiry": "876000h"
}
}
- 生成Etcd客户端证书
cfssl gencert \
-ca=/etc/etcd/ssl/etcd-ca.pem \
-ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
-config=ca-config.json \
-hostname=127.0.0.1,k8s-master-01,k8s-master-02,k8s-master-03,192.168.31.101,192.168.31.102,192.168.31.103 \
-profile=kubernetes \
etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd
# hostname: etcd节点主机名和ip地址, 如果需要扩容etcd则多预留几个主机名和ip
# 执行结果
2022/05/17 21:04:41 [INFO] generate received request
2022/05/17 21:04:41 [INFO] received CSR
2022/05/17 21:04:41 [INFO] generating key: rsa-2048
2022/05/17 21:04:41 [INFO] encoded CSR
2022/05/17 21:04:41 [INFO] signed certificate with serial number 374416837578176400852307196163272282609639254096
- 将证书复制到其他节点
MasterNodes='k8s-master-02 k8s-master-03'
WorkNodes='k8s-node-01 k8s-node-02 k8s-node-03'
for NODE in $MasterNodes; do
ssh $NODE "mkdir -p /etc/etcd/ssl"
for FILE in etcd-ca-key.pem etcd-ca.pem etcd-key.pem etcd.pem; do
scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}
done
done
k8s组件证书
1 根证书生成
- Master01生成kubernetes根证书,然后用根证书去申请其他组件的证书,ca为证书颁发机构
[root@k8s-master-01 pki]# cd /data/src/k8s-ha-install/pki
[root@k8s-master-01 pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca
2022/05/17 21:16:11 [INFO] generating a new CA key and certificate from CSR
2022/05/17 21:16:11 [INFO] generate received request
2022/05/17 21:16:11 [INFO] received CSR
2022/05/17 21:16:11 [INFO] generating key: rsa-2048
2022/05/17 21:16:11 [INFO] encoded CSR
2022/05/17 21:16:11 [INFO] signed certificate with serial number 356110579670536937029012243009570774526723248422
[root@k8s-master-01 pki]# ll /etc/kubernetes/pki/
total 12
-rw-r--r-- 1 root root 1025 May 17 21:16 ca.csr
-rw------- 1 root root 1675 May 17 21:16 ca-key.pem
-rw-r--r-- 1 root root 1411 May 17 21:16 ca.pem
[root@k8s-master-01 pki]# cat ca-csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "Kubernetes",
"OU": "Kubernetes-manual"
}
],
"ca": {
"expiry": "876000h"
}
}
2 生成apiserver证书
cfssl gencert -ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-hostname=10.244.0.1,192.168.31.100,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.31.101,192.168.31.102,192.168.31.103 \
-profile=kubernetes apiserver-csr.json \
| cfssljson -bare /etc/kubernetes/pki/apiserver
# 10.244.0.1是k8s service的网段，如果说需要更改k8s service网段，那就需要更改10.244.0.1，
# 如果不是高可用集群，192.168.31.100为Master01的IP
# 执行结果
2022/05/17 21:29:45 [INFO] generate received request
2022/05/17 21:29:45 [INFO] received CSR
2022/05/17 21:29:45 [INFO] generating key: rsa-2048
2022/05/17 21:29:45 [INFO] encoded CSR
2022/05/17 21:29:45 [INFO] signed certificate with serial number 44589346984861915136867175777203942181126965371
- 生成apiserver的聚合证书。
聚合证书用于给第三方组件使用, 权限比较小
cfssl gencert \
-initca front-proxy-ca-csr.json \
| cfssljson -bare /etc/kubernetes/pki/front-proxy-ca
# 执行结果
2022/05/17 21:38:09 [INFO] generating a new CA key and certificate from CSR
2022/05/17 21:38:09 [INFO] generate received request
2022/05/17 21:38:09 [INFO] received CSR
2022/05/17 21:38:09 [INFO] generating key: rsa-2048
2022/05/17 21:38:09 [INFO] encoded CSR
2022/05/17 21:38:09 [INFO] signed certificate with serial number 94744298379229994126568334765473831074836354862
cfssl gencert \
-ca=/etc/kubernetes/pki/front-proxy-ca.pem \
-ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem \
-config=ca-config.json \
-profile=kubernetes front-proxy-client-csr.json \
| cfssljson -bare /etc/kubernetes/pki/front-proxy-client
# 执行结果, 警告可忽略
2022/05/17 21:39:01 [INFO] generate received request
2022/05/17 21:39:01 [INFO] received CSR
2022/05/17 21:39:01 [INFO] generating key: rsa-2048
2022/05/17 21:39:01 [INFO] encoded CSR
2022/05/17 21:39:01 [INFO] signed certificate with serial number 363789832426414262439760026013971715042182270063
2022/05/17 21:39:01 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
3 controlle-manager证书配置
# 1. 生成controller-manager证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager
# -ca 根证书公钥,
# -ca-key 根证书私钥
# -config 证书签名请求文件
# manager-csr.json controller-manager的证书签名请求文件
# -bare 生成证书的存放路径与文件头
[root@k8s-master-01 pki]# ls -l /etc/kubernetes/pki/controller-manager*
-rw-r--r-- 1 root root 1082 May 17 21:57 /etc/kubernetes/pki/controller-manager.csr
-rw------- 1 root root 1679 May 17 21:57 /etc/kubernetes/pki/controller-manager-key.pem
-rw-r--r-- 1 root root 1501 May 17 21:57 /etc/kubernetes/pki/controller-manager.pem
# 2. 生成controlle-manager.kubeconfig, 用于访问apiserver
# 注意，如果不是高可用集群，192.168.31.100:16443改为master01的地址，16443改为apiserver的端口，默认是6443
# 2.1 set-cluster：设置一个集群项，设置apiserver地址,导入根证书
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.168.31.100:16443 \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# --certificate ca根证书
# --embed-certs=true 导入证书到kubeconfig
# --server apiserver的vip和端口
# --kubeconfig kubeconfig的存放路径
# 文件内容
[root@k8s-master-01 pki]# cat /etc/kubernetes/controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1RENDQXN5Z0F3SUJBZ0lVUG1DTStLMUtXa0hxMm1SYW1FY2dTYmlTVVNZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2R6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEV6QVJCZ05WQkFvVENrdDFZbVZ5Ym1WMFpYTXhHakFZQmdOVkJBc1RFVXQxWW1WeWJtVjBaWE10CmJXRnVkV0ZzTVJNd0VRWURWUVFERXdwcmRXSmxjbTVsZEdWek1DQVhEVEl5TURVeE56RXpNVEV3TUZvWUR6SXgKTWpJd05ESXpNVE14TVRBd1dqQjNNUXN3Q1FZRFZRUUdFd0pEVGpFUU1BNEdBMVVFQ0JNSFFtVnBhbWx1WnpFUQpNQTRHQTFVRUJ4TUhRbVZwYW1sdVp6RVRNQkVHQTFVRUNoTUtTM1ZpWlhKdVpYUmxjekVhTUJnR0ExVUVDeE1SClMzVmlaWEp1WlhSbGN5MXRZVzUxWVd3eEV6QVJCZ05WQkFNVENtdDFZbVZ5Ym1WMFpYTXdnZ0VpTUEwR0NTcUcKU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFNwMjl5N2JKZXdlcW5tNmNUOEFDTjJ2bzh2b3BhN0JVego3dkNEVHJLM1N5V0JuSzQ5YnFKRDljWElxMWpzTmZQTCtRU3V3eG1nU3JJbWF1S0Y2RkUvalhXbStWMWhjN0ZZCjlldlFKV09rSHVLVllXUkMzWG50R3M4MSs5bStxRlY0eFllT2xqVlBEQ2FOK3Ntb2RML1lvY3o0QjNDVzNMbFEKcElEaVRxK2xZRTlKZDljcDF4NkF2cVpxVWdrNUJsVzFYeGNDTEU3aGU1ZlpyQUlBakRIa05wM1NXV2R0YjNZZQowWS9HbHZSWldCZkVOaEdRcE9KM3dFVlVXM1hyN1I1K3JEK3huUlZydk4xUlE0bUhnYW1XbzBEMituNUFrQWRCCkpRTFh3QnF3VHREZmRnQ09ra05ZTVpUeWpFemozNUxGUVltenREMVlua2JVaUhybEQxMHhBZ01CQUFHalpqQmsKTUE0R0ExVWREd0VCL3dRRUF3SUJCakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VDTUIwR0ExVWREZ1FXQkJTUgpncmlqdm1BbW1oYXZTcjdtcTBURVdsQTgxakFmQmdOVkhTTUVHREFXZ0JTUmdyaWp2bUFtbWhhdlNyN21xMFRFCldsQTgxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBbitUQ2ZkdUVxTTk2QncycEk5aFZmMGpEWkh2bmRHeGwKRENwWFJZOXlrKzU1elUwZWhObWdhQ0gxREJXc2JQMWlQMUw1bjVnUXRDV0hjWVZZRkVMcVcrMFFYdEJVd21TMAp0VStnbkhiTy8xWWw0OG9oMG1YcmVPYnozQWJYeG5CRHRXQSt0amdtYy9ReDhhQkkzS1VQd1hUMjVqMnJycGNBCk56OWdwSHU4QmFEdGI1VWJ0K3JhaTZCTEtUVGQrNnZzZnozeG96ekNrSmdTcjYxeXFvd0V1RWFFQmlNY0hIOGUKZ0hNYm1UY216VTJuRmNiWUlmbk91RXVZSVVkQkxmWkhmM00vYUIreFdNdnM0OUVxQlFqNzloSEVkTERaK3BNSApPZkhwMGVoKzlMU0hLeXB6TWVEdUt3UC80U3crK0pmOXE0SWFENEdYaHMwV1cwL1lzRmpXZVE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
server: https://192.168.31.100:16443
name: kubernetes
contexts: null
current-context: ""
kind: Config
preferences: {}
users: null
# 2.2 设置一个环境项，一个上下文, 相当于配置了一个用户
kubectl config set-context system:kube-controller-manager@kubernetes \
--cluster=kubernetes \
--user=system:kube-controller-manager \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# --cluster 集群项name
# --user 用户名
# --kubeconfig kubeconfig文件地址
# 文件内容
[root@k8s-master-01 pki]# cat /etc/kubernetes/controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1RENDQXN5Z0F3SUJBZ0lVUG1DTStLMUtXa0hxMm1SYW1FY2dTYmlTVVNZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2R6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEV6QVJCZ05WQkFvVENrdDFZbVZ5Ym1WMFpYTXhHakFZQmdOVkJBc1RFVXQxWW1WeWJtVjBaWE10CmJXRnVkV0ZzTVJNd0VRWURWUVFERXdwcmRXSmxjbTVsZEdWek1DQVhEVEl5TURVeE56RXpNVEV3TUZvWUR6SXgKTWpJd05ESXpNVE14TVRBd1dqQjNNUXN3Q1FZRFZRUUdFd0pEVGpFUU1BNEdBMVVFQ0JNSFFtVnBhbWx1WnpFUQpNQTRHQTFVRUJ4TUhRbVZwYW1sdVp6RVRNQkVHQTFVRUNoTUtTM1ZpWlhKdVpYUmxjekVhTUJnR0ExVUVDeE1SClMzVmlaWEp1WlhSbGN5MXRZVzUxWVd3eEV6QVJCZ05WQkFNVENtdDFZbVZ5Ym1WMFpYTXdnZ0VpTUEwR0NTcUcKU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFNwMjl5N2JKZXdlcW5tNmNUOEFDTjJ2bzh2b3BhN0JVego3dkNEVHJLM1N5V0JuSzQ5YnFKRDljWElxMWpzTmZQTCtRU3V3eG1nU3JJbWF1S0Y2RkUvalhXbStWMWhjN0ZZCjlldlFKV09rSHVLVllXUkMzWG50R3M4MSs5bStxRlY0eFllT2xqVlBEQ2FOK3Ntb2RML1lvY3o0QjNDVzNMbFEKcElEaVRxK2xZRTlKZDljcDF4NkF2cVpxVWdrNUJsVzFYeGNDTEU3aGU1ZlpyQUlBakRIa05wM1NXV2R0YjNZZQowWS9HbHZSWldCZkVOaEdRcE9KM3dFVlVXM1hyN1I1K3JEK3huUlZydk4xUlE0bUhnYW1XbzBEMituNUFrQWRCCkpRTFh3QnF3VHREZmRnQ09ra05ZTVpUeWpFemozNUxGUVltenREMVlua2JVaUhybEQxMHhBZ01CQUFHalpqQmsKTUE0R0ExVWREd0VCL3dRRUF3SUJCakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VDTUIwR0ExVWREZ1FXQkJTUgpncmlqdm1BbW1oYXZTcjdtcTBURVdsQTgxakFmQmdOVkhTTUVHREFXZ0JTUmdyaWp2bUFtbWhhdlNyN21xMFRFCldsQTgxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBbitUQ2ZkdUVxTTk2QncycEk5aFZmMGpEWkh2bmRHeGwKRENwWFJZOXlrKzU1elUwZWhObWdhQ0gxREJXc2JQMWlQMUw1bjVnUXRDV0hjWVZZRkVMcVcrMFFYdEJVd21TMAp0VStnbkhiTy8xWWw0OG9oMG1YcmVPYnozQWJYeG5CRHRXQSt0amdtYy9ReDhhQkkzS1VQd1hUMjVqMnJycGNBCk56OWdwSHU4QmFEdGI1VWJ0K3JhaTZCTEtUVGQrNnZzZnozeG96ekNrSmdTcjYxeXFvd0V1RWFFQmlNY0hIOGUKZ0hNYm1UY216VTJuRmNiWUlmbk91RXVZSVVkQkxmWkhmM00vYUIreFdNdnM0OUVxQlFqNzloSEVkTERaK3BNSApPZkhwMGVoKzlMU0hLeXB6TWVEdUt3UC80U3crK0pmOXE0SWFENEdYaHMwV1cwL1lzRmpXZVE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
server: https://192.168.31.100:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:kube-controller-manager
name: system:kube-controller-manager@kubernetes
current-context: ""
kind: Config
preferences: {}
users: null
# 2.3 set-credentials 设置一个用户项, 配置用户使用哪个证书生成kubeconfig.导入controller-manager证书
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=/etc/kubernetes/pki/controller-manager.pem \
--client-key=/etc/kubernetes/pki/controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# --client-certificate controller-manager证书公钥
# -client-key controller-manager证书私钥
# --embed-certs 是否导入证书
# --kubeconfig kubeconfig文件地址
# 文件内容
[root@k8s-master-01 pki]# cat /etc/kubernetes/controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ1RENDQXN5Z0F3SUJBZ0lVUG1DTStLMUtXa0hxMm1SYW1FY2dTYmlTVVNZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2R6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEV6QVJCZ05WQkFvVENrdDFZbVZ5Ym1WMFpYTXhHakFZQmdOVkJBc1RFVXQxWW1WeWJtVjBaWE10CmJXRnVkV0ZzTVJNd0VRWURWUVFERXdwcmRXSmxjbTVsZEdWek1DQVhEVEl5TURVeE56RXpNVEV3TUZvWUR6SXgKTWpJd05ESXpNVE14TVRBd1dqQjNNUXN3Q1FZRFZRUUdFd0pEVGpFUU1BNEdBMVVFQ0JNSFFtVnBhbWx1WnpFUQpNQTRHQTFVRUJ4TUhRbVZwYW1sdVp6RVRNQkVHQTFVRUNoTUtTM1ZpWlhKdVpYUmxjekVhTUJnR0ExVUVDeE1SClMzVmlaWEp1WlhSbGN5MXRZVzUxWVd3eEV6QVJCZ05WQkFNVENtdDFZbVZ5Ym1WMFpYTXdnZ0VpTUEwR0NTcUcKU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFNwMjl5N2JKZXdlcW5tNmNUOEFDTjJ2bzh2b3BhN0JVego3dkNEVHJLM1N5V0JuSzQ5YnFKRDljWElxMWpzTmZQTCtRU3V3eG1nU3JJbWF1S0Y2RkUvalhXbStWMWhjN0ZZCjlldlFKV09rSHVLVllXUkMzWG50R3M4MSs5bStxRlY0eFllT2xqVlBEQ2FOK3Ntb2RML1lvY3o0QjNDVzNMbFEKcElEaVRxK2xZRTlKZDljcDF4NkF2cVpxVWdrNUJsVzFYeGNDTEU3aGU1ZlpyQUlBakRIa05wM1NXV2R0YjNZZQowWS9HbHZSWldCZkVOaEdRcE9KM3dFVlVXM1hyN1I1K3JEK3huUlZydk4xUlE0bUhnYW1XbzBEMituNUFrQWRCCkpRTFh3QnF3VHREZmRnQ09ra05ZTVpUeWpFemozNUxGUVltenREMVlua2JVaUhybEQxMHhBZ01CQUFHalpqQmsKTUE0R0ExVWREd0VCL3dRRUF3SUJCakFTQmdOVkhSTUJBZjhFQ0RBR0FRSC9BZ0VDTUIwR0ExVWREZ1FXQkJTUgpncmlqdm1BbW1oYXZTcjdtcTBURVdsQTgxakFmQmdOVkhTTUVHREFXZ0JTUmdyaWp2bUFtbWhhdlNyN21xMFRFCldsQTgxakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBbitUQ2ZkdUVxTTk2QncycEk5aFZmMGpEWkh2bmRHeGwKRENwWFJZOXlrKzU1elUwZWhObWdhQ0gxREJXc2JQMWlQMUw1bjVnUXRDV0hjWVZZRkVMcVcrMFFYdEJVd21TMAp0VStnbkhiTy8xWWw0OG9oMG1YcmVPYnozQWJYeG5CRHRXQSt0amdtYy9ReDhhQkkzS1VQd1hUMjVqMnJycGNBCk56OWdwSHU4QmFEdGI1VWJ0K3JhaTZCTEtUVGQrNnZzZnozeG96ekNrSmdTcjYxeXFvd0V1RWFFQmlNY0hIOGUKZ0hNYm1UY216VTJuRmNiWUlmbk91RXVZSVVkQkxmWkhmM00vYUIreFdNdnM0OUVxQlFqNzloSEVkTERaK3BNSApPZkhwMGVoKzlMU0hLeXB6TWVEdUt3UC80U3crK0pmOXE0SWFENEdYaHMwV1cwL1lzRmpXZVE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
server: https://192.168.31.100:8443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: system:kube-controller-manager
name: system:kube-controller-manager@kubernetes
current-context: ""
kind: Config
preferences: {}
users:
- name: system:kube-controller-manager
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVKakNDQXc2Z0F3SUJBZ0lVZmhZdWx1TnNWS0ZoQWNQNkRiUU5WejdCK29Rd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2R6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEV6QVJCZ05WQkFvVENrdDFZbVZ5Ym1WMFpYTXhHakFZQmdOVkJBc1RFVXQxWW1WeWJtVjBaWE10CmJXRnVkV0ZzTVJNd0VRWURWUVFERXdwcmRXSmxjbTVsZEdWek1DQVhEVEl5TURVeE56RXpOVE13TUZvWUR6SXgKTWpJd05ESXpNVE0xTXpBd1dqQ0JuekVMTUFrR0ExVUVCaE1DUTA0eEVEQU9CZ05WQkFnVEIwSmxhV3BwYm1jeApFREFPQmdOVkJBY1RCMEpsYVdwcGJtY3hKekFsQmdOVkJBb1RIbk41YzNSbGJUcHJkV0psTFdOdmJuUnliMnhzClpYSXRiV0Z1WVdkbGNqRWFNQmdHQTFVRUN4TVJTM1ZpWlhKdVpYUmxjeTF0WVc1MVlXd3hKekFsQmdOVkJBTVQKSG5ONWMzUmxiVHByZFdKbExXTnZiblJ5YjJ4c1pYSXRiV0Z1WVdkbGNqQ0NBU0l3RFFZSktvWklodmNOQVFFQgpCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFPRml6NjlTWU1CYTZmTUR0eWU5cVdTQ0dmWDd0Z0xYZ1FlTnlTWVp0Y0s2CnM4MWM4djhYWEQ4cW5qMnZudEVOZnRTcnl6VHBGem9oS2psUmdIUkRNTFV1Rkhuc244Tjl4d2ErNXMxMWtkMnQKdmpiYjJKekc3M3pNUGpNNXJxNEFsbGl3ZVZFeU1Hd0pyOWx4UW5YbXBOUmpPUHQ4Ti9FWUhJUmtzbEJuQVVzcQo4WDJmQVVIWkU2MENuZG8wZjRTQnNUR2QwUnFNNVJOY2NaVHJHYTJFUDNLUUZNbUZGbEk5T2hubGY4UmUveCtjCjQ5VTdBNk0rckk1eE1QZUlld2Jzd25vM2FvdDRMQ3dxR00rZnluaUh3ZjU4ZmlZQUw1aC9oU1A2SllqcDNqZFEKWlVVVWhZb1BZaDFManNFUktBWUluMVBqRnZiQXh6Vy80Q1ZWV2JGS1VvMENBd0VBQWFOL01IMHdEZ1lEVlIwUApBUUgvQkFRREFnV2dNQjBHQTFVZEpRUVdNQlFHQ0NzR0FRVUZCd01CQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CCkFmOEVBakFBTUIwR0ExVWREZ1FXQkJTWXBkazYrcUY1S3dWaWt3SlZJQzMwTjExWUpEQWZCZ05WSFNNRUdEQVcKZ0JTUmdyaWp2bUFtbWhhdlNyN21xMFRFV2xBODFqQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFrekEzNUpHcwphSFp2UlM0ZE03ZVZkVmVvWGEwVEFxUXlURVlnRkFNU3VudjVaUC9FZ2VKT1NJR2VpbmZ3VUM5WDFEZXdlV2l3CmFEQjhObkF0bTVDQW5kKzR6Z0FqRmZlM0RqY1pUZEdZL1V5Q1k1MDB3T2pDR2lMRlFCdVBncWFHa2ZWRFNxQnAKQm1JOVVWS2VpWDZRb3JraTV4akd5YlRQZWh2WmZ4TTRKZllwNlp3aXdnQWlOMTNRZVBpMXZ6blhFZms2TVdmUwpFTjhLNnpOUm9DVDRCTlVaNzk1K2tkQjd5Rm9LaEEvdWQ1a2h1dU02WlVhcHE0R3dCb2FZMWdEUXl1ODhaM1pwCjVmTnVobTJjSW9QSUIzdnYwbktjUjd2ejg4dERWaWtBQjJwcElwZnBRUVZOREh0Z0NXWEQ2aTlMV1BxL2RCb2wKQno0cWcyUzRrS1JrVHc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBNFdMUHIxSmd3RnJwOHdPM0o3MnBaSUlaOWZ1MkF0ZUJCNDNKSmhtMXdycXp6Vnp5Ci94ZGNQeXFlUGErZTBRMSsxS3ZMTk9rWE9pRXFPVkdBZEVNd3RTNFVlZXlmdzMzSEJyN216WFdSM2EyK050dlkKbk1idmZNdytNem11cmdDV1dMQjVVVEl3YkFtdjJYRkNkZWFrMUdNNCszdzM4UmdjaEdTeVVHY0JTeXJ4Zlo4QgpRZGtUclFLZDJqUi9oSUd4TVozUkdvemxFMXh4bE9zWnJZUS9jcEFVeVlVV1VqMDZHZVYveEY3L0g1emoxVHNECm96NnNqbkV3OTRoN0J1ekNlamRxaTNnc0xDb1l6NS9LZUlmQi9ueCtKZ0F2bUgrRkkvb2xpT25lTjFCbFJSU0YKaWc5aUhVdU93UkVvQmdpZlUrTVc5c0RITmIvZ0pWVlpzVXBTalFJREFRQUJBb0lCQVFDUEhCdTlPZlJmRHhzUAppQ05xNDMzRWFPdXRDaGZHOUNsa3IzMnlhSTdGeDZEVlhCaWJLcTBUR3ErQmdacTVLUFdJZWxDOEZ1ajlxd05SCmc5T3BmdVJWbHAyLzBTU2NqNmVwTzl2M1I1aklCa01LT0V2eE9Fdm1sUlZGbDNHMzRIMldjTytIUS9RRkZaMkgKMXVlWlQwc0g1THpReWs0SEV3VkFkMlBWczZIWnBJaUtXajExTUE5QklhV0xOZmxkOXE3dkZSTDJPWmtXcGZPSQpZM2hyWXMranMzVUNwQmM4V0RWTkFlR3YyZVJTUTVKd2VQZzRaZVcyd3VRR1dNa2YwaUk3aXQ1RDN5WExPdmdaClhhOXlDU3hUd0JpaXZvQ3o2K2FmVXZnS3M4VWVPZG1Jc1VxL1BIZlpWZkdKckRreWtjMkdvL0szb0c0MTJPYjQKc28wWTFpRkJBb0dCQVBuL2VxYUFUWnZWVzlSOEZNMGVqTjRmMzBtYUZpeGNHODFTUWk5L1lKTStqNHo0VmxsTgpXKytUdW1QWCs3akx5dUluMDZGd25OUGN6RjFaTzB2UWkvT1Q2UjZqb1VweTYzclg4V1I2VFA2OVAvb1Ivc3BSCnJOZkxiamRiMEo5bG9qME5WeG9icjQ1OE1tTWN6MHBxZzFWK3ozVUFLdjQvakUrcm5ScHBzUHp4QW9HQkFPYk0KRUVuQmNYU0FMeDhTTE5WMVhkbkxlSVVuOVcvYmFjd0JnVGxHd2dkNFROV3B0NlRldmRWbkJyZFhVU1RaQXBpbApNZFUrSGZ4RnlOd1p0RUE0QUN4bm16bzlLR3BNMWp3L2o4Z000UzZzamRrZmtmbXlBcWNUWUh1MWw2NUt2UEVQCitaYkRGY0hLT0JRWnY0bTNDY2drRWhTRzV2Ulpld0JwaEx0ME1WOWRBb0dBV2hyWDFRMG5hOFJCdmRzZkVETXoKcUplcVBmZjRoL0tHM3NFSU0rQmdLWklCNFZoY253RS80cURITEZkYlZlYTE4RDlVaXJwdysvZDMvU2s5TXYwdQpoQk5La1kxK2c3dlozY1BaTUZMWVQzUmNpOEJTcWc4NEVlc3poV1psVWg0cWxJQ3JaVENYWE82c3BvWnF2REtaCnRZWG9OZzVpY0pMcytvWXJNS3JwYkxFQ2dZRUE1SUVwcHp5RkRlbFR2aG1LbGhUTHhMUzFNSEN0aWYvY3NZVFQKNGxkeUIxOU9BMFV6YzJLczVMcEtaZjluY1dvQ0xndHdXVVpVL2M1QjNkajlJNC9PYkNod0FhdEhkbWQ0dk5IWgpreUZkV1k2eUtrUWRqUEIzdTk5dGFVNFRUUmJtRm0zUW1UbXhNdHI1eHJ6dmJIUHlsVnRSSTAybElFdnZnaXIyCnBYbVc0R2tDZ1lFQW4yQTl2ZDd0RTRKSC9kRWc1aVZHZFFKR1pKazQzWWcvdW1HcHM4Mk9laXhxSkh1bVJGb0sKUnNteDZtRE5FeFdHbG1sc3JuaGEzMTlVTjlqQlM3WGFtSEQvMW5reUtxVDArOEplQXJ5MWZlQXkvWmJ0Y3luaApPM29CbUs2TkpxQmJJWjk4MllKNkZ3d2RveTJ1NDRHQ0wvb2pOQjlSYU9Nd0Nwb1krQk5lV0J3PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
# 2.4 使用某个环境当做默认环境
kubectl config use-context system:kube-controller-manager@kubernetes \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# 执行完后controller-manager.kubeconfig文件中current-context值为system:kube-controller-manager@kubernetes
4 scheduler证书配置
# 1 生成scheduler证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler
# 2 生成scheduler.kubeconfig, 过程和参数与controller-manager类似
# 2.1 配置apiserver地址, 导入根证书
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.168.31.100:16443 \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# 2.2 配置用户项,导入scheduler证书
kubectl config set-credentials system:kube-scheduler \
--client-certificate=/etc/kubernetes/pki/scheduler.pem \
--client-key=/etc/kubernetes/pki/scheduler-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# 2.3 配置上下文, 设置用户
kubectl config set-context system:kube-scheduler@kubernetes \
--cluster=kubernetes \
--user=system:kube-scheduler \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# 2.4 使用某个环境当做默认环境
kubectl config use-context system:kube-scheduler@kubernetes \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
5 管理员证书配置
用于kubectl管理集群使用的证书
# 1 生成admin证书
cfssl gencert \
-ca=/etc/kubernetes/pki/ca.pem \
-ca-key=/etc/kubernetes/pki/ca-key.pem \
-config=ca-config.json \
-profile=kubernetes \
admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin
# 2 生成admin.kubeconfig
# 注意，如果不是高可用集群，192.168.31.100:16443改为master01的地址，16443改为apiserver的端口，默认是6443
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.168.31.100:16443 \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-credentials kubernetes-admin \
--client-certificate=/etc/kubernetes/pki/admin.pem \
--client-key=/etc/kubernetes/pki/admin-key.pem \
--embed-certs=true \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config set-context kubernetes-admin@kubernetes \
--cluster=kubernetes \
--user=kubernetes-admin \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
kubectl config use-context kubernetes-admin@kubernetes \
--kubeconfig=/etc/kubernetes/admin.kubeconfig
# kubectl 使用admin.kubeconfig连接对应的集群
kubectl --kubeconfig /etc/kubernetes/admin.kubeconfig
6 生成一个秘钥对
创建ServiceAccount的时候会生成一个对应的secret
这个secret里面是有token的
controller-manager会基于这个密钥对来生成ServiceAccount对应secret里面的token
# 生成私钥
openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
# 生成公钥
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub
7 发送证书到其他Master节点
for NODE in k8s-master-02 k8s-master-03; do
for FILE in $(ls /etc/kubernetes/pki | grep -v etcd); do
scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE};
done;
for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig; do
scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE};
done;
done
- 查看证书文件
[root@k8s-master-01 pki]# ll /etc/kubernetes/pki/
total 92
-rw-r--r-- 1 root root 1025 May 17 22:30 admin.csr
-rw------- 1 root root 1679 May 17 22:30 admin-key.pem
-rw-r--r-- 1 root root 1444 May 17 22:30 admin.pem
-rw-r--r-- 1 root root 1029 May 17 21:29 apiserver.csr
-rw------- 1 root root 1679 May 17 21:29 apiserver-key.pem
-rw-r--r-- 1 root root 1692 May 17 21:29 apiserver.pem
-rw-r--r-- 1 root root 1025 May 17 21:16 ca.csr
-rw------- 1 root root 1675 May 17 21:16 ca-key.pem
-rw-r--r-- 1 root root 1411 May 17 21:16 ca.pem
-rw-r--r-- 1 root root 1082 May 17 21:57 controller-manager.csr
-rw------- 1 root root 1679 May 17 21:57 controller-manager-key.pem
-rw-r--r-- 1 root root 1501 May 17 21:57 controller-manager.pem
-rw-r--r-- 1 root root 891 May 17 21:38 front-proxy-ca.csr
-rw------- 1 root root 1679 May 17 21:38 front-proxy-ca-key.pem
-rw-r--r-- 1 root root 1143 May 17 21:38 front-proxy-ca.pem
-rw-r--r-- 1 root root 903 May 17 21:39 front-proxy-client.csr
-rw------- 1 root root 1679 May 17 21:39 front-proxy-client-key.pem
-rw-r--r-- 1 root root 1188 May 17 21:39 front-proxy-client.pem
-rw-r--r-- 1 root root 1675 May 17 22:39 sa.key
-rw-r--r-- 1 root root 451 May 17 22:40 sa.pub
-rw-r--r-- 1 root root 1058 May 17 22:21 scheduler.csr
-rw------- 1 root root 1679 May 17 22:21 scheduler-key.pem
-rw-r--r-- 1 root root 1476 May 17 22:21 scheduler.pem
[root@k8s-master-01 pki]# ls /etc/kubernetes/pki/ | wc -l
23
8 ETCD配置
etcd配置大致相同，注意修改每个Master节点的etcd配置的主机名和IP地址
Master01
mkdir -p /data/kubernetes/etcd/wal
cat >>/etc/etcd/etcd.config.yml<<EOF
name: 'k8s-master-01'
data-dir: /data/kubernetes/etcd
wal-dir: /data/kubernetes/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.31.101:2380'
listen-client-urls: 'https://192.168.31.101:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.31.101:2380'
advertise-client-urls: 'https://192.168.31.101:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-01=https://192.168.31.101:2380,k8s-master-02=https://192.168.31.102:2380,k8s-master-03=https://192.168.31.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
Master02
mkdir -p /data/kubernetes/etcd/wal
cat >>/etc/etcd/etcd.config.yml<<EOF
name: 'k8s-master-02'
data-dir: /data/kubernetes/etcd
wal-dir: /data/kubernetes/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.31.102:2380'
listen-client-urls: 'https://192.168.31.102:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.31.102:2380'
advertise-client-urls: 'https://192.168.31.102:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-01=https://192.168.31.101:2380,k8s-master-02=https://192.168.31.102:2380,k8s-master-03=https://192.168.31.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
Master03
mkdir -p /data/kubernetes/etcd/wal
cat >>/etc/etcd/etcd.config.yml<<EOF
name: 'k8s-master-03'
data-dir: /data/kubernetes/etcd
wal-dir: /data/kubernetes/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.31.103:2380'
listen-client-urls: 'https://192.168.31.103:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.31.103:2380'
advertise-client-urls: 'https://192.168.31.103:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-01=https://192.168.31.101:2380,k8s-master-02=https://192.168.31.102:2380,k8s-master-03=https://192.168.31.103:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
创建Service, 三个节点相同
cat >>/usr/lib/systemd/system/etcd.service<<EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
所有Master节点创建etcd的证书目录
mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
启动ETCD
systemctl daemon-reload
systemctl enable --now etcd
查看etcd状态
export ETCDCTL_API=3
etcdctl --endpoints="192.168.31.103:2379,192.168.31.102:2379,192.168.31.101:2379" \
--cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem \
--cert=/etc/kubernetes/pki/etcd/etcd.pem \
--key=/etc/kubernetes/pki/etcd/etcd-key.pem \
endpoint \
status \
--write-out=table
+---------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 192.168.31.103:2379 | 353538130ceca90b | 3.5.1 | 20 kB | false | false | 2 | 9 | 9 | |
| 192.168.31.102:2379 | 778de9861c23dc52 | 3.5.1 | 20 kB | false | false | 2 | 9 | 9 | |
| 192.168.31.101:2379 | f53c0ada9cafb29 | 3.5.1 | 20 kB | true | false | 2 | 9 | 9 | |
+---------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
9 Kubernetes组件配置
9.1 Apiserver
- 所有节点创建相关目录
mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes
注意本文档使用的k8s service网段为10.244.0.0/16，该网段不能和宿主机的网段、Pod网段的重复，请按需修改
9.1.1 Master01 配置
cat >>/usr/lib/systemd/system/kube-apiserver.service<<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--v=2 \
--logtostderr=true \
--allow-privileged=true \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--insecure-port=0 \
--advertise-address=192.168.31.101 \
--service-cluster-ip-range=10.244.0.0/16 \
--service-node-port-range=30000-32767 \
--etcd-servers=https://192.168.31.101:2379,https://192.168.31.102:2379,https://192.168.31.103:2379 \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--client-ca-file=/etc/kubernetes/pki/ca.pem \
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/pki/sa.pub \
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
# --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
# 注意修改
# --advertise-address 本机IP
# --service-cluster-ip-range Service网段
# --etcd-servers etcd集群ip和端口
9.1.2 Master02 配置
cat >>/usr/lib/systemd/system/kube-apiserver.service<<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--v=2 \
--logtostderr=true \
--allow-privileged=true \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--insecure-port=0 \
--advertise-address=192.168.31.102 \
--service-cluster-ip-range=10.244.0.0/16 \
--service-node-port-range=30000-32767 \
--etcd-servers=https://192.168.31.101:2379,https://192.168.31.102:2379,https://192.168.31.103:2379 \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--client-ca-file=/etc/kubernetes/pki/ca.pem \
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/pki/sa.pub \
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
# --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
# 注意修改
# --advertise-address 本机IP
# --service-cluster-ip-range Service网段
# --etcd-servers etcd集群ip和端口
9.1.3 Master03配置
cat >>/usr/lib/systemd/system/kube-apiserver.service<<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-apiserver \
--v=2 \
--logtostderr=true \
--allow-privileged=true \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--insecure-port=0 \
--advertise-address=192.168.31.103 \
--service-cluster-ip-range=10.244.0.0/16 \
--service-node-port-range=30000-32767 \
--etcd-servers=https://192.168.31.101:2379,https://192.168.31.102:2379,https://192.168.31.103:2379 \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--client-ca-file=/etc/kubernetes/pki/ca.pem \
--tls-cert-file=/etc/kubernetes/pki/apiserver.pem \
--tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem \
--kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \
--kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/pki/sa.pub \
--service-account-signing-key-file=/etc/kubernetes/pki/sa.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota \
--authorization-mode=Node,RBAC \
--enable-bootstrap-token-auth=true \
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
--proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem \
--requestheader-allowed-names=aggregator \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User
# --token-auth-file=/etc/kubernetes/token.csv
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
EOF
# 注意修改
# --advertise-address 本机IP
# --service-cluster-ip-range Service网段
# --etcd-servers etcd集群ip和端口
9.1.4 启动apiserver
# 所有Master节点开启kube-apiserver
systemctl daemon-reload && systemctl enable --now kube-apiserver
# 检查kube-server状态
[root@k8s-master-01 pki]# systemctl status kube-apiserver
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since Sat 2020-08-22 21:26:49 CST; 26s ago
- 如果系统日志有这些提示可以忽略
Dec 11 20:51:15 k8s-master-01 kube-apiserver: I1211 20:51:15.004739 7450 clientconn.go:948] ClientConn switching balancer to "pick_first"
Dec 11 20:51:15 k8s-master-01 kube-apiserver: I1211 20:51:15.004843 7450 balancer_conn_wrappers.go:78] pickfirstBalancer: HandleSubConnStateChange: 0xc011bd4c80, {CONNECTING <nil>}
Dec 11 20:51:15 k8s-master-01 kube-apiserver: I1211 20:51:15.010725 7450 balancer_conn_wrappers.go:78] pickfirstBalancer: HandleSubConnStateChange: 0xc011bd4c80, {READY <nil>}
Dec 11 20:51:15 k8s-master-01 kube-apiserver: I1211 20:51:15.011370 7450 controlbuf.go:508] transport: loopyWriter.run returning. connection error: desc = "transport is closing"
9.2 ControllerManager
所有Master节点配置kube-controller-manager service（所有master节点配置一样）
9.2.1 创建kube-controller-manager service
vim /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
--v=2 \
--logtostderr=true \
--address=127.0.0.1 \
--root-ca-file=/etc/kubernetes/pki/ca.pem \
--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \
--service-account-private-key-file=/etc/kubernetes/pki/sa.key \
--kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \
--leader-elect=true \
--cluster-signing-duration=876000h0m0s \
--use-service-account-credentials=true \
--node-monitor-grace-period=40s \
--node-monitor-period=5s \
--pod-eviction-timeout=2m0s \
--controllers=*,bootstrapsigner,tokencleaner \
--allocate-node-cidrs=true \
--cluster-cidr=10.244.0.0/16 \
--requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \
--node-cidr-mask-size=24
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
# 注意修改
# --cluster-cidr=10.244.0.0/16 规划的pod网段
9.2.2 所有Master节点启动kube-controller-manager
systemctl daemon-reload
systemctl enable --now kube-controller-manager
# 检查启动状态
[root@k8s-master-01 pki]# systemctl status kube-controller-manager
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2022-05-17 23:33:03 CST; 13s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 14210 (kube-controller)
Tasks: 7
Memory: 25.0M
CGroup: /system.slice/kube-controller-manager.service
└─14210 /usr/local/bin/kube-controller-manager --v=2 --logtostderr=true --address=127.0.0.1 --root-ca-file=/etc/kubernetes/pki/ca.pem --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem --cluster-signing-key-file=/etc/kuberne...
9.3 Scheduler
9.3.1 创建kube-scheduler service
所有master节点配置一样
cat >>/usr/lib/systemd/system/kube-scheduler.service <<EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-scheduler \
--v=2 \
--logtostderr=true \
--address=127.0.0.1 \
--leader-elect=true \
--kubeconfig=/etc/kubernetes/scheduler.kubeconfig
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
9.3.2 所有Master节点启动scheduler
systemctl daemon-reload
systemctl enable --now kube-scheduler
# 检查状态
[root@k8s-master-01 data]# systemctl status kube-scheduler
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2022-05-17 23:45:24 CST; 21s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 15069 (kube-scheduler)
Tasks: 9
Memory: 20.3M
CGroup: /system.slice/kube-scheduler.service
└─15069 /usr/local/bin/kube-scheduler --v=2 --logtostderr=true --address=127.0.0.1 --leader-elect=true --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
10 TLS Bootstrapping 配置
TLS Bootstrapping用于kubelet证书自动颁发, kubelet证书与其他组件的证书不同, 手动生成比较麻烦
只需要在Master01创建bootstrap
# 注意，如果不是高可用集群，192.168.31.100:16443改为master01的地址，16443改为apiserver的端口，默认是6443
10.1 生成Bootstrapping.kubeconfig文件
cd /data/src/k8s-ha-install/bootstrap
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.168.31.100:16443 \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-credentials tls-bootstrap-token-user \
--token=c8ad9c.2e4d610cf3e7426e \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-context tls-bootstrap-token-user@kubernetes \
--cluster=kubernetes \
--user=tls-bootstrap-token-user \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config use-context tls-bootstrap-token-user@kubernetes \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
10.2 创建bootstrapping secret
注意：如果要修改bootstrap.secret.yaml的token-id和token-secret，需要保证下图红圈内的字符串一致的，并且位数是一样的。还要保证上一步 kubectl config set-credentials的–token：c8ad9c.2e4d610cf3e7426e与你修改后的字符串要一致
10.2.1 配置kubectl默认的kubeconfig
mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config
# 可以使用kubectl查询集群信息才可以继续,否则不行，需要排查k8s组件是否有故障
[root@k8s-master-01 bootstrap]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-2 Healthy {"health":"true","reason":""}
etcd-1 Healthy {"health":"true","reason":""}
etcd-0 Healthy {"health":"true","reason":""}
10.2.2 创建secret
[root@k8s-master-01 bootstrap]# kubectl create -f bootstrap.secret.yaml
secret/bootstrap-token-c8ad9c created
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-certificate-rotation created
clusterrole.rbac.authorization.k8s.io/system:kube-apiserver-to-kubelet created
clusterrolebinding.rbac.authorization.k8s.io/system:kube-apiserver created
11 Node节点配置
11.1 复制证书
# Master01节点复制证书至Node节点
for NODE in k8s-master-02 k8s-master-03 k8s-node-01 k8s-node-02 k8s-node-03; do
ssh $NODE mkdir -p /etc/kubernetes/pki /etc/etcd/ssl /etc/etcd/ssl
for FILE in etcd-ca.pem etcd.pem etcd-key.pem; do
scp /etc/etcd/ssl/$FILE $NODE:/etc/etcd/ssl/
done
for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig; do
scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}
done
done
11.2 kubelet配置
10.2.1 所有节点创建相关目录
mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/
10.2.2 所有节点配置kubelet service
所有节点配置相同
cat >>/usr/lib/systemd/system/kubelet.service<<EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
[Service]
ExecStart=/usr/local/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
- 如果Runtime为Containerd，请使用如下Kubelet的配置：
# Runtime为Containerd
cat >> /etc/systemd/system/kubelet.service.d/10-kubelet.conf<<EOF
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/usr/local/cni/bin --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock --cgroup-driver=systemd"
Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml"
Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
ExecStart=
ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS
EOF
- 如果Runtime为Docker，请使用如下Kubelet的配置：
# Runtime为Docker
cat >> /etc/systemd/system/kubelet.service.d/10-kubelet.conf<<
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/usr/local/cni/bin"
Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6"
Environment="KUBELET_EXTRA_ARGS=--node-labels=node.kubernetes.io/node='' "
ExecStart=
ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_SYSTEM_ARGS \$KUBELET_EXTRA_ARGS
EOF
- 创建kubelet的配置文件
注意：如果更改了k8s的service网段，需要更改kubelet-conf.yml 的clusterDNS:配置，改成k8s Service网段的第十个地址，比如192.168.0.10
cat >>/etc/kubernetes/kubelet-conf.yml<<EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.244.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
- 启动所有节点kubelet
systemctl daemon-reload;systemctl enable --now kubelet
- 此时系统日志/var/log/messages显示只有如下信息为正常
Unable to update cni config: no networks found in /etc/cni/net.d
或者
Container runtime network not ready
如果有很多报错日志，或者有大量看不懂的报错，说明kubelet的配置有误，需要检查kubelet配置
11.3 kube-proxy配置
11.3.1 生成kube-proxy.kubeconfig
Master01上操作
cd /data/src/k8s-ha-install
# 1. 创建kube-proxy serviceaccount
kubectl -n kube-system create serviceaccount kube-proxy
# 2. 创建clusterrolebinding
kubectl create clusterrolebinding system:kube-proxy \
--clusterrole system:node-proxier \
--serviceaccount kube-system:kube-proxy
# 3.获取SECRET Name
SECRET=$(kubectl -n kube-system get sa/kube-proxy \
--output=jsonpath='{.secrets[0].name}')
# 4.获取token
JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET \
--output=jsonpath='{.data.token}' | base64 -d)
PKI_DIR=/etc/kubernetes/pki
K8S_DIR=/etc/kubernetes
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=https://192.168.31.100:16443 \
--kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig
kubectl config set-credentials kubernetes \
--token=${JWT_TOKEN} \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config set-context kubernetes \
--cluster=kubernetes \
--user=kubernetes \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config use-context kubernetes \
--kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
11.3.2 将kubeconfig发送至其他节点
for NODE in k8s-master-02 k8s-master-03; do
scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
done
for NODE in k8s-node-01 k8s-node-02 k8s-node-03; do
scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig
done
11.3.3 所有节点添加kube-proxy的配置和service文件：
cat >> /usr/lib/systemd/system/kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
ExecStart=/usr/local/bin/kube-proxy \
--config=/etc/kubernetes/kube-proxy.yaml \
--v=2
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target
EOF
11.3.4所有节点创建kube-proxy.yaml
如果更改了集群Pod的网段，需要更改kube-proxy.yaml的clusterCIDR为自己的Pod网段：
cat >> /etc/kubernetes/kube-proxy.yaml<<EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
qps: 5
clusterCIDR: 10.96.0.0/16
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
ipvs:
masqueradeAll: true
minSyncPeriod: 5s
scheduler: "rr"
syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF
11.3.5 所有节点启动kube-proxy
systemctl daemon-reload;systemctl enable --now kube-proxy
systemctl status kube-proxy
日志显示Adding new service port即安装成功
12 Calico 安装
12.1 安装官方推荐版本
Master01 操作
cd /data/src/k8s-ha-install/calico/
# 更改calico的网段，主要需要将红色部分的网段，改为自己的Pod网段
sed -i "s#POD_CIDR#10.96.0.0/16#g" calico.yaml
# 检查网段是自己的Pod网段，
[root@k8s-master-01 calico]# grep "IPV4POOL_CIDR" calico.yaml -A 1
- name: CALICO_IPV4POOL_CIDR
value: "10.96.0.0/16"
# 安装calico
kubectl apply -f calico.yaml
- 查看容器状态
[root@k8s-master-01 calico]# kubectl get po -n kube-system -owide |grep calico
calico-kube-controllers-6f6595874c-mpg9q 1/1 Running 0 83s
calico-node-7pnmj 1/1 Running 0 84s
calico-node-b7kmn 1/1 Running 0 83s
calico-node-h6pnx 1/1 Running 0 83s
calico-node-lkmwq 1/1 Running 0 83s
calico-node-tsdft 1/1 Running 0 83s
calico-typha-6b6cf8cbdf-5g4dq 1/1 Running 0 84s
# 如果容器状态异常可以使用kubectl describe 或者kubectl logs查看容器的日志
13 CoreDNS安装
13.1 安装官方推荐版本（推荐）
cd /data/src/k8s-ha-install/
# 如果更改了k8s service的网段需要将coredns的serviceIP改成k8s service网段的第十个IP
COREDNS_SERVICE_IP=`kubectl get svc | grep kubernetes | awk '{print $3}'`0
sed -i "s#KUBEDNS_SERVICE_IP#${COREDNS_SERVICE_IP}#g" CoreDNS/coredns.yaml
# 安装coredns
[root@k8s-master-01 k8s-ha-install]# kubectl create -f CoreDNS/coredns.yaml
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created
# 查看容器状态
[root@k8s-master-01 k8s-ha-install]# kubectl get po -A | grep coredns
kube-system coredns-5db5696c7-crm4l 1/1 Running 0 2m31s
13.2 安装最新版CoreDNS
# 获取Coredns ip
COREDNS_SERVICE_IP=`kubectl get svc | grep kubernetes | awk '{print $3}'`0
# 获取最新coredis部署文件
mkdir /data/src/coredns_new
git clone https://github.com/coredns/deployment.git
cd deployment/kubernetes
# 部署
[root@k8s-master-01 kubernetes]# ./deploy.sh -s -i ${COREDNS_SERVICE_IP} | kubectl apply -f -
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created
查看状态
# kubectl get po -n kube-system -l k8s-app=kube-dns
NAME READY STATUS RESTARTS AGE
coredns-85b4878f78-h29kh 1/1 Running 0 8h
14 安装Metrics Server
在新版的Kubernetes中系统资源的采集均使用Metrics-server，可以通过Metrics采集节点和Pod的内存、磁盘、CPU和网络的使用率。
14.1 安装metrics
cd /data/src/k8s-ha-install/metrics-server
kubectl create -f .
# 等待metrics server启动然后查看状态
[root@k8s-master-01 metrics-server]# kubectl get po -n kube-system | grep metrics
metrics-server-6bf7dcd649-gdk7h 1/1 Running 0 55s
[root@k8s-master-01 metrics-server]# kubectl top node
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
k8s-master-01 140m 3% 2261Mi 28%
k8s-master-02 105m 2% 1240Mi 15%
k8s-master-03 130m 3% 1215Mi 15%
k8s-node-01 54m 1% 639Mi 8%
k8s-node-02 40m 1% 629Mi 8%
15 安装Dashboard
15.1 安装指定版本dashboard
cd /data/src/k8s-ha-install/dashboard/
kubectl create -f .
15.2 安装最新版本
官方GitHub地址：https://github.com/kubernetes/dashboard
安装
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.1/aio/deploy/recommended.yaml
创建管理员用户
cat >>admin.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kube-system
EOF
更改dashboard的svc为NodePort
kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
查看svc端口号
kubectl get svc kubernetes-dashboard -n kubernetes-dashboard
根据自己的实例端口号，通过任意安装了kube-proxy的宿主机的IP+端口即可访问到dashboard：
查看token
用于访问Dashboard
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')
访问Dashboard
- 浏览器打开https://192.168.31.101:31286, 输入token
16 集群验证
16.1 安装busybox
mkdir -p /data/src/test
cd /data/src/test
cat >>busybox_test.yaml<<EOF
apiVersion: v1
kind: Pod
metadata:
name: busybox
namespace: default
spec:
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
restartPolicy: Always
EOF
kubectl apply -f busybox_test.yaml
16.2 验证点
# 1. Pod必须能解析Service
[root@k8s-master-01 test]# kubectl exec busybox -n default -- nslookup kubernetes
Server: 10.244.0.10
Address 1: 10.244.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.244.0.1 kubernetes.default.svc.cluster.local
# 2. Pod必须能解析跨namespace的Service
[root@k8s-master-01 test]# kubectl exec busybox -n default -- nslookup kube-dns.kube-system
Server: 10.244.0.10
Address 1: 10.244.0.10 kube-dns.kube-system.svc.cluster.local
Name: kube-dns.kube-system
Address 1: 10.244.0.10 kube-dns.kube-system.svc.cluster.local
# 3. 每个节点都必须要能访问Kubernetes的kubernetes svc 443和kube-dns的service 53
[root@k8s-master-01 test]# telnet 10.244.0.1 443
Trying 10.244.0.1...
Connected to 10.244.0.1.
Escape character is '^]'.
[root@k8s-master-01 test]# telnet 10.244.0.10 53
Trying 10.244.0.10...
Connected to 10.244.0.10.
Escape character is '^]'.
# 4. Pod和Pod之前要能通
[root@k8s-master-01 test]# kubectl get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 0 29m 10.96.95.1 k8s-master-02 <none> <none>
[root@k8s-master-01 test]# kubectl get po -A -owide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default busybox 1/1 Running 0 30m 10.96.95.1 k8s-master-02 <none> <none>
kube-system calico-kube-controllers-6f6595874c-bd5tl 1/1 Running 0 38m 10.96.151.129 k8s-master-01 <none> <none>
kube-system calico-node-2lzgd 1/1 Running 0 38m 192.168.31.106 k8s-node-03 <none> <none>
kube-system calico-node-mtvr7 1/1 Running 0 38m 192.168.31.101 k8s-master-01 <none> <none>
kube-system calico-node-phzkq 1/1 Running 0 38m 192.168.31.103 k8s-master-03 <none> <none>
kube-system calico-node-pk5xd 1/1 Running 0 38m 192.168.31.104 k8s-node-01 <none> <none>
kube-system calico-node-q299h 1/1 Running 0 38m 192.168.31.105 k8s-node-02 <none> <none>
kube-system calico-node-t9vb2 1/1 Running 0 38m 192.168.31.102 k8s-master-02 <none> <none>
kube-system calico-typha-6b6cf8cbdf-bbvgs 1/1 Running 0 38m 192.168.31.101 k8s-master-01 <none> <none>
kube-system coredns-5db5696c7-v96g8 1/1 Running 0 32m 10.96.154.193 k8s-node-01 <none> <none>
kube-system metrics-server-6bf7dcd649-d9kj7 1/1 Running 0 31m 10.96.44.193 k8s-node-02 <none> <none>
kubernetes-dashboard dashboard-metrics-scraper-799d786dbf-mmcj5 1/1 Running 0 20m 10.96.183.129 k8s-master-03 <none> <none>
kubernetes-dashboard kubernetes-dashboard-fb8648fd9-7nsxg 1/1 Running 0 20m 10.96.89.129 k8s-node-03 <none> <none>
# a) 同namespace能通信
# b) 跨namespace能通信
# c) 跨机器能通信
[root@k8s-master-01 test]# kubectl exec -it busybox -- sh
/ # ping 10.96.44.193
PING 10.96.44.193 (10.96.44.193): 56 data bytes
64 bytes from 10.96.44.193: seq=0 ttl=62 time=0.584 ms
64 bytes from 10.96.44.193: seq=1 ttl=62 time=0.338 ms
64 bytes from 10.96.44.193: seq=2 ttl=62 time=0.290 ms
64 bytes from 10.96.44.193: seq=3 ttl=62 time=0.324 ms
^C
--- 10.96.44.193 ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max = 0.290/0.384/0.584 ms
## 创建deploy
[root@k8s-master-01 test]# kubectl create deploy nginx --image=nginx --replicas 3
deployment.apps/nginx created
[root@k8s-master-01 test]# kubectl get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
nginx 0/3 3 0 23s
[root@k8s-master-01 test]# kubectl get pods -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 0 49m 10.96.95.1 k8s-master-02 <none> <none>
nginx-85b98978db-n5dts 1/1 Running 0 95s 10.96.183.131 k8s-master-03 <none> <none>
nginx-85b98978db-swnzf 1/1 Running 0 95s 10.96.154.195 k8s-node-01 <none> <none>
nginx-85b98978db-xjgc9 1/1 Running 0 95s 10.96.89.130 k8s-node-03 <none> <none>
[root@k8s-master-01 test]# curl 10.96.183.131:80 -I
HTTP/1.1 200 OK
Server: nginx/1.21.6
Date: Wed, 18 May 2022 15:11:26 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Tue, 25 Jan 2022 15:03:52 GMT
Connection: keep-alive
ETag: "61f01158-267"
Accept-Ranges: bytes
17 集群关键配置
Docker配置
# 采用containerd作为Runtime无需配置
vim /etc/docker/daemon.json
{ "registry-mirrors": [
"https://registry.docker-cn.com",
"http://hub-mirror.c.163.com",
"https://docker.mirrors.ustc.edu.cn"
],
"exec-opts": ["native.cgroupdriver=systemd"],
"max-concurrent-downloads": 10,
"max-concurrent-uploads": 5,
"log-opts": {
"max-size": "300m",
"max-file": "2"
},
"live-restore": true
}
# "max-concurrent-downloads": 10, 并发下载
# "max-concurrent-uploads": 5, 并发上传
# "registry-mirrors": [] 镜像地址
# "log-opts": { 日志选项
# "max-size": "300m", 单个日志文件大小
# "max-file": "2" 最多保存日志文件个数
# },
# "live-restore": true 热重启, 默认false,在重启docker的时候会使所有容器也重启, 为true的时候重启docker不会影响容器
kube-controller-manager配置
Bootstrapping颁发证书过期时间
vim /usr/lib/systemd/system/kube-controller-manager.service
# 此配置已经默认为true, 不需要配置
# --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true
将此配置写入kube-controller-manager.service
--cluster-signing-duration=876000h0m0s
- 重启kube-controller-manager
systemctl daemon-reload;systemctl restart kube-controller-manager
修改集群的加密算法, 防止扫描出安全问题
vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf
# 加入下面两个参数
# 加密算法
--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
# 下载镜像deadline, 防止pod频繁报错ImagePullBackOff
--image-pull-progress-deadline=30m"
- 重启kubelet
systemctl daemon-reload;systemctl restart kubelet
添加Role标签
# 添加标签之前
[root@k8s-master-01 test]# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-master-01 Ready <none> 107m v1.23.6
k8s-master-02 Ready <none> 104m v1.23.6
k8s-master-03 Ready <none> 104m v1.23.6
k8s-node-01 Ready <none> 104m v1.23.6
k8s-node-02 Ready <none> 104m v1.23.6
k8s-node-03 Ready <none> 104m v1.23.6
# 给节点添加标签
kubectl label node k8s-master-01 node-role.kubernetes.io/master=''
kubectl label node k8s-master-02 node-role.kubernetes.io/master=''
kubectl label node k8s-master-03 node-role.kubernetes.io/master=''
kubectl label node k8s-node-01 node-role.kubernetes.io/node=''
kubectl label node k8s-node-02 node-role.kubernetes.io/node=''
kubectl label node k8s-node-03 node-role.kubernetes.io/node=''
# 添加标签之后
[root@k8s-master-01 test]# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-master-01 Ready master 111m v1.23.6
k8s-master-02 Ready master 108m v1.23.6
k8s-master-03 Ready master 108m v1.23.6
k8s-node-01 Ready node 108m v1.23.6
k8s-node-02 Ready node 108m v1.23.6
k8s-node-03 Ready node 108m v1.23.6
K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容
更多推荐
1
0- 0
已为社区贡献2条内容
所有评论(0)