kubernetes【k8s】adm方式安装[超级详细]
集群规划【1master节点2node节点】主机名IP地址推荐配置k8s-master192.168.91.1321C2G40Gk8s-node1192.168.91.1331C2G40Gk8s-node2192.168.91.1341C2G40G网络规划networkIP地址段解释NodeIP192.168.91.0
集群规划【1master节点2node节点】
主机名 IP地址 推荐配置
k8s-master 172.16.115.237 1C2G40G
k8s-node1 172.16.115.238 1C2G40G
k8s-node2 172.16.115.239 1C2G40G
网络规划
network IP地址段 解释
NodeIP 172.16.115.0/24 #对外提供用户访问
PodIP 10.2.0.0/16 #集群内部IP,可以动态感知后面的POD IP
ClusterIP 10.1.0.0/16 #POD的IP
1.k8s环境准备【机器标准化】所有节点执行
1.1配置hosts解析
cat >> /etc/hosts << EOF
172.16.115.237 k8s-master
172.16.115.238 k8s-node1
172.16.115.239 k8s-node2
EOF
1.2关闭防火墙
systemctl stop firewalld NetworkManager
systemctl disable firewalld NetworkManager
1.3关闭SELinux
setenforce 0
sed -i 's#SELINUX=disabled#SELINUX=disabled#g' /etc/selinux/config
getenforce
1.4关闭SWAP分区
swapoff -a
sed -i '/swap/d' /etc/fstab
free -h
1.5配置时间同步
yum install chrony -y
systemctl start chronyd
systemctl enable chronyd
date
1.6下载Docker CE Yum 源
yum-config-manager \
--add-repo https://download.docker.com/linux/centos/docker-ce.repo
1.7安装开源版Docker CE
yum install docker-ce-19.03.6 docker-ce-cli-19.03.6 containerd.io
1.8启动Docker
service docker start
chkconfig docker on #开机启动
1.9查看Docker版本
docker version
docker info
1.10配置镜像加速器
mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://plqjafsr.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker
2.Kubernetes 1.15 安装
2.1初始化工具安装
yum install net-tools vim wget lrzsz git -y
2.2设置免密码登录
yum install -y expect
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
export mypass=123456
name=(k8s-master k8s-node1 k8s-node2)
for i in ${name[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
expect {
\"*yes/no*\" {send \"yes\r\"; exp_continue}
\"*password*\" {send \"$mypass\r\"; exp_continue}
\"*Password*\" {send \"$mypass\r\";}
}"
done
2.3连接测试
ssh k8s-node1
ssh k8s-node2
2.4优化内核参数
cat >>/etc/sysctl.conf<<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
fs.file-max=52706963
fs.nr_open=52706963
EOF
2.5应用内核配置
modprobe br_netfilter
sysctl -p
3配置证书
3.1下载自签名证书生成工具
mkdir /soft && cd /soft
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo
3.2生成ETCD证书
CA 证书配置
mkdir /root/etcd && cd /root/etcd
cat << EOF | tee ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
创建CA证书请求文件
cat << EOF | tee ca-csr.json
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
创建ETCD证书请求文件
#可以把所有的master IP 加入到csr文件中(Master-1)
cat << EOF | tee server-csr.json
{
"CN": "etcd",
"hosts": [
"k8s-master",
"k8s-master2",
"k8s-master3",
"172.16.115.237",
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
生成 ETCD CA 证书和ETCD公私钥(Master-1)
cd /root/etcd/
生成ca证书(Master-1)
cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
ll
total 24
-rw-r--r-- 1 root root 287 Apr 5 11:23 ca-config.json #ca 的配置文件
-rw-r--r-- 1 root root 956 Apr 5 11:26 ca.csr #ca 证书生成文件
-rw-r--r-- 1 root root 209 Apr 5 11:23 ca-csr.json #ca 证书请求文件
-rw------- 1 root root 1679 Apr 5 11:26 ca-key.pem #ca 证书key
-rw-r--r-- 1 root root 1265 Apr 5 11:26 ca.pem #ca 证书
-rw-r--r-- 1 root root 338 Apr 5 11:26 server-csr.json
生成etcd证书(Master-1)
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
ll
total 36
-rw-r--r-- 1 root root 287 Apr 5 11:23 ca-config.json
-rw-r--r-- 1 root root 956 Apr 5 11:26 ca.csr
-rw-r--r-- 1 root root 209 Apr 5 11:23 ca-csr.json
-rw------- 1 root root 1679 Apr 5 11:26 ca-key.pem
-rw-r--r-- 1 root root 1265 Apr 5 11:26 ca.pem
-rw-r--r-- 1 root root 1054 Apr 5 11:31 server.csr
-rw-r--r-- 1 root root 338 Apr 5 11:26 server-csr.json
-rw------- 1 root root 1675 Apr 5 11:31 server-key.pem #etcd客户端使用
-rw-r--r-- 1 root root 1379 Apr 5 11:31 server.pem
创建 Kubernetes 相关证书
#此证书用于Kubernetes节点直接的通信, 与之前的ETCD证书不同. (Master-1)
配置ca 文件(Master-1)
mkdir /root/kubernetes/ && cd /root/kubernetes/
cat << EOF | tee ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
创建ca证书申请文件(Master-1)
cat << EOF | tee ca-csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成API SERVER证书申请文件(Master-1)
cat << EOF | tee server-csr.json
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"10.0.0.2",
"172.16.115.237",
"172.16.115.238",
"172.16.115.239",
"k8s-master",
"k8s-node1",
"k8s-node2",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
创建 Kubernetes Proxy 证书申请文件(Master-1)
cat << EOF | tee kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成 kubernetes CA 证书和公私钥
生成ca证书(Master-1)
cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
生成 api-server 证书(Master-1)
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
生成 kube-proxy 证书(Master-1)
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
部署ETCD
下载etcd二进制安装文件
mkdir -p /soft && cd /soft
wget https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz
tar -xvf etcd-v3.3.10-linux-amd64.tar.gz
cd etcd-v3.3.10-linux-amd64/
cp etcd etcdctl /usr/local/bin/
编辑etcd配置文件
#注意修改每个节点的ETCD_NAME
#注意修改每个节点的监听地址
mkdir -p /etc/etcd/{cfg,ssl}
cat >/etc/etcd/cfg/etcd.conf<<EOFL
#[Member]
ETCD_NAME="master-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.115.237:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.115.237:2379,http://172.16.115.237:2390"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.115.237:2380" #如果有多个可以用,号分割
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.115.237:2379"
ETCD_INITIAL_CLUSTER="master-1=https://172.16.115.237:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOFL
创建ETCD的系统启动服务(所有master)
cat > /usr/lib/systemd/system/etcd.service<<EOFL
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/cfg/etcd.conf
ExecStart=/usr/local/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--peer-cert-file=/etc/etcd/ssl/server.pem \
--peer-key-file=/etc/etcd/ssl/server-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOFL
复制etcd证书到指定目录
mkdir -p /etc/etcd/ssl/
cp /root/etcd/*pem /etc/etcd/ssl/ -rf
复制etcd证书到每个节点
for i in k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/etcd/{cfg,ssl};done
for i in k8s-node1 k8s-node2;do scp /etc/etcd/ssl/* $i:/etc/etcd/ssl/;done
for i in k8s-node1 k8s-node2;do echo $i "------>"; ssh $i ls /etc/etcd/ssl;done
启动etcd (所有节点)
chkconfig etcd on
service etcd start
service etcd status
检查etcd 集群是否运行正常
etcdctl --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem --endpoints="https://k8s-master:2379" cluster-health
创建Docker所需分配POD 网段 (任意master节点)
etcdctl --ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/server.pem --key-file=/etc/etcd/ssl/server-key.pem \
--endpoints="https://k8s-master:2379" \
set /coreos.com/network/config \
'{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
检查是否建立网段
etcdctl \
--endpoints=https://k8s-master:2379 \
--ca-file=/etc/etcd/ssl/ca.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--cert-file=/etc/etcd/ssl/server.pem \
get /coreos.com/network/config
4部署Flannel
下载Flannel二进制包
所有的节点,下载到master-1
mkdir /soft ; cd /soft
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
tar xvf flannel-v0.10.0-linux-amd64.tar.gz
mv flanneld mk-docker-opts.sh /usr/local/bin/
复制flanneld到其他的所有节点
for i in k8s-node1 k8s-node2;do scp /usr/local/bin/flanneld $i:/usr/local/bin/;done
for i in k8s-node1 k8s-node2;do scp /usr/local/bin/mk-docker-opts.sh $i:/usr/local/bin/;done
配置Flannel (所有节点)
mkdir -p /etc/flannel
cat > /etc/flannel/flannel.cfg<<EOF
FLANNEL_OPTIONS="-etcd-endpoints=https://172.16.115.237:2379 -etcd-cafile=/etc/etcd/ssl/ca.pem -etcd-certfile=/etc/etcd/ssl/server.pem -etcd-keyfile=/etc/etcd/ssl/server-key.pem --healthz-ip=0.0.0.0 --healthz-port=7100"
EOF
#多个ETCD: -etcd-endpoints=https://172.16.115.237:2379,https://172.16.115.238:2379,https://172.16.115.239:2379
配置Flannel配置文件
cat > /usr/lib/systemd/system/flanneld.service <<EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/etc/flannel/flannel.cfg
ExecStart=/usr/local/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
启动Flannel
service flanneld start
chkconfig flanneld on
service flanneld status
所有的节点都需要有172.17.0.0/16 网段IP
ip a | grep flannel
节点停止flanneld
service flanneld stop
修改Docker启动文件
cat >/usr/lib/systemd/system/docker.service<<EOFL
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOFL
重启Docker服务
systemctl daemon-reload
service flanneld restart
service docker restart
检查IP地址, docker 与flanneld 是同一个网段
ip a
在每个Node节点Ping其他的节点, 网段都是通的。
[root@k8s-node1 ~]# ping 172.17.95.1
PING 172.17.95.1 (172.17.95.1) 56(84) bytes of data.
64 bytes from 172.17.95.1: icmp_seq=1 ttl=64 time=0.322 ms
64 bytes from 172.17.95.1: icmp_seq=2 ttl=64 time=0.225 ms
172.17.95.164 bytes from 172.17.95.1: icmp_seq=3 ttl=64 time=0.232 ms
64 bytes from 172.17.95.1: icmp_seq=4 ttl=64 time=0.245 ms
5安装Master 组件
Master端需要安装的组件如下:
kube-apiserver
kube-scheduler
kube-controller-manager
安装Api Server服务
下载Kubernetes二进制包(1.15.1)(master-1)
cd /soft
wget https://dl.k8s.io/v1.15.10/kubernetes-server-linux-amd64.tar.gz
tar xvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-scheduler kube-apiserver kube-controller-manager kubectl /usr/local/bin/
配置Kubernetes证书
#Kubernetes各个组件之间通信需要证书,需要复制个每个master节点(master-1)
mkdir -p /etc/kubernetes/{cfg,ssl}
cp /root/kubernetes/*.pem /etc/kubernetes/ssl/
复制到其他的节点
for i in k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/kubernetes/{cfg,ssl};done
for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done
for i in k8s-node1 k8s-node2;do echo $i "---------->"; ssh $i ls /etc/kubernetes/ssl;done
创建 TLS Bootstrapping Token
# TLS bootstrapping 功能就是让 kubelet 先使用一个预定的低权限用户连接到 apiserver,
然后向 apiserver 申请证书,kubelet 的证书由 apiserver 动态签署
#Token可以是任意的包涵128 bit的字符串,可以使用安全的随机数发生器生成
head -c 16 /dev/urandom | od -An -t x | tr -d ' '
b78d1a70a76180a848090a178c806229
编辑Token 文件(master-1)
#b78d1a70a76180a848090a178c806229:随机字符串,自定义生成; kubelet-bootstrap:用户名; 10001:UID; system:kubelet-bootstrap:用户组
echo b78d1a70a76180a848090a178c806229,kubelet-bootstrap,10001,"system:kubelet-bootstrap" /etc/kubernetes/cfg/token.csv
#如有其他msater节点请复制到其他的master节点
创建Apiserver配置文件(所有的master节点)
#配置文件内容基本相同, 如果有多个节点, 那么需要修改IP地址即可
cat >/etc/kubernetes/cfg/kube-apiserver.cfg <<EOFL
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=8080 \
--etcd-servers=https://172.16.115.237:2379 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=0.0.0.0 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/etc/kubernetes/ssl/server.pem \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/server.pem \
--etcd-keyfile=/etc/etcd/ssl/server-key.pem"
EOFL
#参数说明
--logtostderr 启用日志
---v 日志等级
--etcd-servers etcd 集群地址
--etcd-servers=https://172.16.115.237:2379
--bind-address 监听地址
--secure-port https 安全端口
--advertise-address 集群通告地址
--allow-privileged 启用授权
--service-cluster-ip-range Service 虚拟IP地址段
--enable-admission-plugins 准入控制模块
--authorization-mode 认证授权,启用RBAC授权
--enable-bootstrap-token-auth 启用TLS bootstrap功能
--token-auth-file token 文件
--service-node-port-range Service Node类型默认分配端口范围
配置kube-apiserver 启动文件(所有的master节点)
cat >/usr/lib/systemd/system/kube-apiserver.service<<EOFL
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.cfg
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOFL
启动kube-apiserver服务
service kube-apiserver start
chkconfig kube-apiserver on
service kube-apiserver status
查看加密的端口是否已经启动
netstat -anltup | grep 6443
查看加密的端口是否已经启动(node节点)
telnet 172.16.115.237 6443
部署kube-scheduler 服务
#创建kube-scheduler配置文件(所有的master节点)
cat >/etc/kubernetes/cfg/kube-scheduler.cfg<<EOFL
KUBE_SCHEDULER_OPTS="--logtostderr=true --v=4 --bind-address=0.0.0.0 --master=127.0.0.1:8080 --leader-elect"
EOFL
查看配置文件
cat /etc/kubernetes/cfg/kube-scheduler.cfg
创建kube-scheduler 启动文件
#创建kube-scheduler systemd unit 文件(所有的master节点)
cat >/usr/lib/systemd/system/kube-scheduler.service<<EOFL
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-scheduler.cfg
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOFL
启动kube-scheduler服务(所有的master节点)
service kube-scheduler restart
chkconfig kube-scheduler on
查看Master节点组件状态(任意一台master)
kubectl get cs
部署kube-controller-manager
创建kube-controller-manager配置文件(所有节点)
cat >/etc/kubernetes/cfg/kube-controller-manager.cfg<<EOFL
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \
--v=4 \
--master=127.0.0.1:8080 \
--leader-elect=true \
--address=0.0.0.0 \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem"
EOFL
参数说明
--master=127.0.0.1:8080 #指定Master地址
--leader-elect #竞争选举机制产生一个 leader 节点,其它节点为阻塞状态。
--service-cluster-ip-range #kubernetes service 指定的IP地址范围。
创建kube-controller-manager 启动文件
cat >/usr/lib/systemd/system/kube-controller-manager.service<<EOFL
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.cfg
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOFL
启动kube-controller-manager服务
chkconfig kube-controller-manager on
service kube-controller-manager start
service kube-controller-manager status
查看Master 节点组件状态
必须要在各个节点组件正常的情况下, 才去部署Node节点组件.(master节点)
kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
6部署Node节点组件
部署 kubelet 组件
从Master节点复制Kubernetes 文件到Node
cd /soft
for i in k8s-node1 k8s-node2;do scp kubernetes/server/bin/kubelet kubernetes/server/bin/kube-proxy $i:/usr/local/bin/;done
创建kubelet bootstrap.kubeconfig 文件
Maste-1节点
mkdir /root/config ; cd /root/config
cat >environment.sh<<EOFL
# 创建kubelet bootstrapping kubeconfig
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=\${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=\${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
#通过 bash environment.sh获取 bootstrap.kubeconfig 配置文件。
EOFL
执行脚本
sh environment.sh
创建kube-proxy kubeconfig文件 (master-1)
cat >env_proxy.sh<<EOF
# 创建kube-proxy kubeconfig文件
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=\${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
EOF
执行脚本
sh env_proxy.sh
复制kubeconfig文件与证书到所有Node节点
将bootstrap kubeconfig kube-proxy.kubeconfig 文件复制到所有Node节点
远程创建目录 (master-1)
for i in k8s-node1 k8s-node2;do ssh $i "mkdir -p /etc/kubernetes/{cfg,ssl}";done
复制证书文件ssl (master-1)
for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done
复制kubeconfig文件 (master-1)
for i in k8s-node1 k8s-node2;do scp -rp bootstrap.kubeconfig kube-proxy.kubeconfig $i:/etc/kubernetes/cfg/;done
创建kubelet参数配置文件
不同的Node节点, 需要修改IP地址 (node节点操作)
cat >/etc/kubernetes/cfg/kubelet.config<<EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 172.16.115.238
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS: ["10.0.0.2"]
clusterDomain: cluster.local.
failSwapOn: false
authentication:
anonymous:
enabled: true
EOF
创建kubelet配置文件
不同的Node节点, 需要修改IP地址
/etc/kubernetes/cfg/kubelet.kubeconfig 文件自动生成
cat >/etc/kubernetes/cfg/kubelet<<EOF
KUBELET_OPTS="--logtostderr=true \
--v=4 \
--hostname-override=172.16.115.239 \
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/etc/kubernetes/cfg/bootstrap.kubeconfig \
--config=/etc/kubernetes/cfg/kubelet.config \
--cert-dir=/etc/kubernetes/ssl \
--pod-infra-container-image=docker.io/kubernetes/pause:latest"
EOF
创建kubelet系统启动文件(node节点)
cat >/usr/lib/systemd/system/kubelet.service<<EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet
ExecStart=/usr/local/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
将kubelet-bootstrap用户绑定到系统集群角色
master-1节点操作
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap
启动kubelet服务(node节点)
chkconfig kubelet on
service kubelet start
service kubelet status
服务端批准与查看CSR请求
查看CSR请求
Maste-1节点操作
kubectl get csr
NAME AGE REQUESTOR CONDITION
node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM 32s kubelet-bootstrap Pending
node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM 30s kubelet-bootstrap Pending
批准请求
Master节点操作
kubectl certificate approve node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM
kubectl certificate approve node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM
节点重名处理
如果出现节点重名, 可以先删除证书, 然后重新申请
Master节点删除csr
kubectl delete csr node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM
Node节点删除kubelet.kubeconfig
客户端重启kubelet服务, 再重新申请证书
rm -rf /etc/kubernetes/cfg/kubelet.kubeconfig
查看节点状态
所有的Node节点状态必须为Ready (master)
kubectl get nodesNAME
STATUS ROLES AGE VERSION
172.16.115.238 Ready <none> 2m37s v1.15.10
172.16.115.239 Ready <none> 2m23s v1.15.10
部署kube-proxy 组件
kube-proxy 运行在所有Node节点上, 监听Apiserver 中 Service 和 Endpoint 的变化情况,创建路由规则来进行服务负载均衡。
创建kube-proxy配置文件
注意修改hostname-override地址, 不同的节点则不同。
cat >/etc/kubernetes/cfg/kube-proxy<<EOF
KUBE_PROXY_OPTS="--logtostderr=true \
--v=4 \
--metrics-bind-address=0.0.0.0 \
--hostname-override=172.16.115.239 \
--cluster-cidr=10.0.0.0/24 \
--kubeconfig=/etc/kubernetes/cfg/kube-proxy.kubeconfig"
EOF
创建kube-proxy systemd unit 文件
cat >/usr/lib/systemd/system/kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
启动kube-proxy 服务
chkconfig kube-proxy on
service kube-proxy start
service kube-proxy status
运行Demo项目
kubectl run nginx --image=nginx --replicas=2
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
deployment.apps/nginx created
获取容器IP与运行节点
kubectl get pods -o wide
创建容器svc端口
kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
查看容器状态
kubectl describe pod nginx-7bb7cd8db5-7ndfk
查看SVC
kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 3h14m
nginx NodePort 10.0.0.129 <none> 88:34947/TCP 94s
访问web
curl http://172.16.115.238:34947
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
删除项目
kubectl delete deployment nginx
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx
服务启动顺序
启动Master节点
service keepalived start
service etcd start
service kube-scheduler start
service kube-controller-manager start
service kube-apiserver restart
kubectl get cs
启动Node节点
service flanneld start
service docker start
service kubelet start
service kube-proxy start
停止Node节点
service kubelet stop
service kube-proxy stop
service docker stop
service flanneld stop
停止Master 节点
service kube-controller-manager stop
service kube-scheduler stop
service etcd stop
service keepalived stop
7部署DNS
部署coredns
cd /soft/kubernetes
tar xf kubernetes-src.tar.gz
mkdir /root/dns && cd /root/dns
cp /soft/kubernetes/cluster/addons/dns/coredns/coredns.yaml.sed /root/dns/coredns.yaml
修改一些参数:
修改参数有3个地方,一个是ip6.arpa 指定,一个是更改成国内镜像源,一个是定义clusterIP,具体如下
ip6.arpa修改为kubernetes cluster.local. in-addr.arpa ip6.arpa
国内镜像修改为 coredns/coredns:1.2.6
clusterIP修改为自己集群设置的IP范围内的,我集群的是 10.0.0.0/24,所以设置为10.0.0.2(且不为已使用的IP)
修改resources:
limits:
memory: 1Gi
requests:
cpu: 1028m
memory: 500Mi
具体的yaml是(我们只需修改clusterIP的IP为自己集群的IP范围内,且不重复的)
作者:凤非飞
链接:https://www.jianshu.com/p/d655e35bf54a
来源:简书
著作权归作者所有。商业转载请联系作者获得授权,非商业转载请注明出处。
kubectl apply -f coredns.yaml
查询所有ns中的pod
kubectl get pod -A
查询指定ns中的pod
kubectl get pod -n kube-system
查看启动进程
kubectl get pod -n kube-system coredns-76fb6cf764-c5tln -n kube-system
查看SVC
kubectl get svc -o wide -n=kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
kube-dns ClusterIP 10.0.0.254 <none> 53/UDP,53/TCP,9153/TCP 87s k8s-app=kube-dns
验证DNS是否有效
删除之前创建的nginx demo
kubectl delete deployment nginx
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx
启动新容器
kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
#出现错误
error: unable to upgrade connection: Forbidden (user=system:anonymous, verb=create, resource=nodes, subresource=proxy)
#解决方法
kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous
kubectl delete pod dnstools
kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
创建Nginx 容器
kubectl run nginx --image=nginx --replicas=2
创建svc (cluster IP)
kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort
查看SVC
kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 4h50m
default nginx NodePort 10.0.0.221 <none> 88:42616/TCP 32m
kube-system kube-dns ClusterIP 10.0.0.2 <none> 53/UDP,53/TCP,9153/TCP 4s
测试解析Nginx
dns 解析的名称是svc (service 名称, 非pod名称)
kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
If you don't see a command prompt, try pressing enter.
dnstools# nslookup nginx
Server: 10.0.0.2
Address: 10.0.0.2#53
Name: nginx.default.svc.cluster.local
Address: 10.0.0.221
案例:容器的网络访问不区分命名空间(kubernetes ns)
在default ns 可以访问到kube-system ns 服务nginx
kubectl run nginx-n1 --image=nginx --replicas=1 -n kube-system
查看容器状态(指定命名空间)
kubectl get pods -n kube-system
查看容器状态(显示所有的命名空间)
kubectl get pod,svc -A
kubectl expose deployment nginx-n1 --port=99 --target-port=80 -n kube-system
跨ns访问服务
kubectl get svc -n kube-system | grep nginx-n1
nginx-n1 ClusterIP 10.0.0.68 <none> 99/TCP 10s
访问服务
curl 10.0.0.68
解析不成功
nslookup nginx-n1
Server: 10.0.0.2
Address: 10.0.0.2#53
** server can't find nginx-n1: NXDOMAIN
解决方法(默认解析为default空间)
nslookup nginx-n1.kube-system.svc.cluster.local
Server: 10.0.0.2
Address: 10.0.0.2#53
Name: nginx-n1.kube-system.svc.cluster.local
Address: 10.0.0.68
部署Dashboard
下载文件
创建目录(master-1节点)
mkdir /root/dashboard
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml
修改端口
修改为nodeport端口50000
注意镜像地址无法下载, 使用另外的镜像替换
mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.1
sed -i '/targetPort:/a\ \ \ \ \ \ nodePort: 50000\n\ \ type: NodePort' kubernetes-dashboard.1.10.yaml
部署
kubectl apply -f kubernetes-dashboard.yaml
查看服务端口
kubectl get services -n kube-system
创建用户授权
kubectl create serviceaccount dashboard-admin -n kube-system
kubectl create clusterrolebinding \
dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
获取Token
kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
Name: dashboard-admin-token-ptxn8
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: dashboard-admin
kubernetes.io/service-account.uid: 2c17e5ee-c11b-4fde-bd0b-02235f433ee0
Type: kubernetes.io/service-account-token
Data
====
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW
4tdG9rZW4tcHR4bjgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiMmMxN2U1ZWUtYzExYi00ZmRlLWJkMGItMDIyMzVmNDMzZWUwIiwic3ViIjoic
3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.i2728HQZ5NRYdwXpSVNnTXK0UEQJ1va6XFyrJMdJ--2QHvV0xzUfF4uEM0HSMnXUFVoLjRZMarGYJ6_4eptoIW7Px0vKUoWY2--SU-ET-pPXKLFBMQ-c8FSKjuMa1W6-fz1Z4oJyt_4VNRbJrzQbhYcQm8EeiVBs-N0pd4qvoUnJqR2
z_TgdctONPKsQCCy7BadzApoGT00TFGvVEjC-4kOD4ZoWXURItK89VgMAmgxxGmxr3WrD_e-lad9Za_CElA_n90eBPOP5JB3yV0iAxsV6uyo2HZagrvHzbfj7hzmCaHPLlvQc4tApDhnN1QY-RHaJKitZiebQ8TUsmT9DRQ
ca.crt: 1359 bytes
namespace: 11 bytes
登录系统
http://nodeip:50000 访问
8.部署kuboard
kubectl apply -f https://kuboard.cn/install-script/kuboard.yaml
获取token
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')
获取nodeport端口
kubectl get svc -n kube-system
kuboard NodePort 10.0.0.95 <none> 80:32567/TCP 15h
登录系统
http://nodeip:32567 访问
9部署Ingress
服务反向代理
部署Traefik 2.0版本
创建 traefik-crd.yaml 文件 (master-1)
cd /root/ingress
cat >/root/ingress/traefik-crd.yaml<<EOF
## IngressRoute
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ingressroutes.traefik.containo.us
spec:
scope: Namespaced
group: traefik.containo.us
version: v1alpha1
names:
kind: IngressRoute
plural: ingressroutes
singular: ingressroute
---
## IngressRouteTCP
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: ingressroutetcps.traefik.containo.us
spec:
scope: Namespaced
group: traefik.containo.us
version: v1alpha1
names:
kind: IngressRouteTCP
plural: ingressroutetcps
singular: ingressroutetcp
---
## Middleware
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: middlewares.traefik.containo.us
spec:
scope: Namespaced
group: traefik.containo.us
version: v1alpha1
names:
kind: Middleware
plural: middlewares
singular: middleware
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: tlsoptions.traefik.containo.us
spec:
scope: Namespaced
group: traefik.containo.us
version: v1alpha1
names:
kind: TLSOption
plural: tlsoptions
singular: tlsoption
EOF
创建Traefik CRD资源(master-1)
cd /root/ingress
kubectl create -f traefik-crd.yaml -n kube-system
kubectl get CustomResourceDefinition
创建Traefik RABC文件(master-1)
cat >/root/ingress/traefik-rbac.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: kube-system
name: traefik-ingress-controller
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
rules:
- apiGroups: [""]
resources: ["services","endpoints","secrets"]
verbs: ["get","list","watch"]
- apiGroups: ["extensions"]
resources: ["ingresses"]
verbs: ["get","list","watch"]
- apiGroups: ["extensions"]
resources: ["ingresses/status"]
verbs: ["update"]
- apiGroups: ["traefik.containo.us"]
resources: ["middlewares"]
verbs: ["get","list","watch"]
- apiGroups: ["traefik.containo.us"]
resources: ["ingressroutes"]
verbs: ["get","list","watch"]
- apiGroups: ["traefik.containo.us"]
resources: ["ingressroutetcps"]
verbs: ["get","list","watch"]
- apiGroups: ["traefik.containo.us"]
resources: ["tlsoptions"]
verbs: ["get","list","watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
name: traefik-ingress-controller
namespace: kube-system
EOF
创建RABC 资源
kubectl create -f traefik-rbac.yaml -n kube-system
创建Traefik ConfigMap (master-1)
cat >/root/ingress/traefik-config.yaml<<EOF
kind: ConfigMap
apiVersion: v1
metadata:
name: traefik-config
data:
traefik.yaml: |-
serversTransport:
insecureSkipVerify: true
api:
insecure: true
dashboard: true
debug: true
metrics:
prometheus: ""
entryPoints:
web:
address: ":80"
websecure:
address: ":443"
providers:
kubernetesCRD: ""
log:
filePath: ""
level: error
format: json
accessLog:
filePath: ""
format: json
bufferingSize: 0
filters:
retryAttempts: true
minDuration: 20
fields:
defaultMode: keep
names:
ClientUsername: drop
headers:
defaultMode: keep
names:
User-Agent: redact
Authorization: drop
Content-Type: keep
EOF
创建Traefik ConfigMap资源配置
kubectl apply -f traefik-config.yaml -n kube-system
设置节点标签
设置节点label
kubectl label nodes 172.16.115.238 IngressProxy=true
kubectl label nodes 172.16.115.239 IngressProxy=true
查看节点标签
检查是否成功
kubectl get nodes --show-labels
NAME STATUS ROLES AGE VERSION LABELS
172.16.115.238 Ready <none> 44h v1.15.10 IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.238,kubernetes.io/os=linux
172.16.115.239 Ready <none> 44h v1.15.10 IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.239,kubernetes.io/os=linux
创建 traefik 部署文件
注意每个Node节点的80与443端口不能被占用
netstat -antupl | grep -E "80|443"
cat >/root/ingress/traefik-deploy.yaml<<EOF
apiVersion: v1
kind: Service
metadata:
name: traefik
spec:
ports:
- name: web
port: 80
- name: websecure
port: 443
- name: admin
port: 8080
selector:
app: traefik
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: traefik-ingress-controller
labels:
app: traefik
spec:
selector:
matchLabels:
app: traefik
template:
metadata:
name: traefik
labels:
app: traefik
spec:
serviceAccountName: traefik-ingress-controller
terminationGracePeriodSeconds: 1
containers:
- image: traefik:v2.0.5
name: traefik-ingress-lb
ports:
- name: web
containerPort: 80
hostPort: 80 #hostPort方式,将端口暴露到集群节点
- name: websecure
containerPort: 443
hostPort: 443 #hostPort方式,将端口暴露到集群节点
- name: admin
containerPort: 8080
resources:
limits:
cpu: 300m
memory: 500Mi
requests:
cpu: 100m
memory: 102Mi
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
args:
- --configfile=/config/traefik.yaml
volumeMounts:
- mountPath: "/config"
name: "config"
volumes:
- name: config
configMap:
name: traefik-config
tolerations: #设置容忍所有污点,防止节点被设置污点
- operator: "Exists"
nodeSelector: #设置node筛选器,在特定label的节点上启动
IngressProxy: "true"
EOF
部署 Traefik 资源
kubectl apply -f traefik-deploy.yaml -n kube-system
kubectl get DaemonSet -A
NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system traefik-ingress-controller 2 2 0 2 0 IngressProxy=true 6s
Traefik 路由配置
配置Traefik Dashboard
cat >/root/ingress/traefik-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
name: traefik-dashboard-route
spec:
entryPoints:
- web
routes:
- match: Host(`traefik.lateautumn4lin.dashboard`)
kind: Rule
services:
- name: traefik
port: 8080
EOF
创建Ingress (traefik)
kubectl apply -f traefik-dashboard-route.yaml -n kube-system
客户端访问Traefik Dashboard
绑定物理主机Hosts文件或者域名解析
/etc/hosts
172.16.115.239 ingress.hostscc.com
访问web
ingress.hostscc.com
部署访问服务(http)
创建https服务
代理dashboard https 服务
创建自签名证书
cd /root/ingress
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=cloud.lateautumn4lin.dashboard"
将证书存储到 Kubernetes Secret中
kubectl create secret generic cloud-mydlq-tls --from-file=tls.crt --from-file=tls.key -n kube-system
查看系统secret
kubectl get secret -n kube-system
dashboard-tls kubernetes.io/tls 2 41s
创建路由文件
先查询kuberbentes dashboard 的命名空间
cat >/root/ingress/kubernetes-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
name: kubernetes-dashboard-route
spec:
entryPoints:
- websecure
tls:
secretName: cloud-mydlq-tls
routes:
- match: Host(`cloud.lateautumn4lin.dashboard`)
kind: Rule
services:
- name: kubernetes-dashboard
port: 443
EOF
创建 Kubernetes Dashboard 路由规则对象
kubectl apply -f kubernetes-dashboard-route.yaml -n kube-system
查看创建的路由
kubectl get IngressRoute -A
绑定hosts 访问
172.16.115.239 cloud.lateautumn4lin.dashboard
获取token
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')
9部署监控系统
监控方案
Prometheus 与 Grafana
安装NFS服务端master节点安装nfs
yum -y install nfs-utils
创建nfs目录
mkdir -p /ifs/kubernetes
修改权限
chmod -R 777 /ifs/kubernetes
编辑export文件
cat >/etc/exports<<EOF
/ifs/kubernetes *(rw,no_root_squash,sync)
EOF
修改配置启动文件
#修改配置文件
cat >/etc/systemd/system/sockets.target.wants/rpcbind.socket<<EOFL
[Unit]
Description=RPCbind Server Activation Socket
[Socket]
ListenStream=/var/run/rpcbind.sock
ListenStream=0.0.0.0:111
ListenDatagram=0.0.0.0:111
[Install]
WantedBy=sockets.target
EOFL
启动rpcbind、nfs服务
systemctl restart rpcbind
systemctl enable rpcbind
systemctl restart nfs
systemctl enable nfs
配置生效
exportfs -f
showmount测试(master-1)
showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *
所有node节点安装客户端
yum -y install nfs-utils
所有的Node检查
showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *
部署PVC
Nfs服务端地址需要修改
mkdir /root/nfs
cat >/root/nfs/nfs-deployment.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
name: nfs-client-provisioner
spec:
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
imagePullPolicy: IfNotPresent
image: quay.io/external_storage/nfs-client-provisioner:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: fuseim.pri/ifs
- name: NFS_SERVER
value: 172.16.115.237
- name: NFS_PATH
value: /ifs/kubernetes
volumes:
- name: nfs-client-root
nfs:
server: 172.16.115.237
path: /ifs/kubernetes
EOF
cat >/root/nfs/nfs-class.yaml<<EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: managed-nfs-storage
provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
archiveOnDelete: "true"
EOF
cat >/root/nfs/nfs-rabc.yaml<<EOF
kind: ServiceAccount
apiVersion: v1
metadata:
name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
EOF
kubectl apply -f nfs-class.yaml
kubectl apply -f nfs-deployment.yaml
kubectl apply -f nfs-rabc.yaml
查看nfs pod状态
kubectl get pods
NAME READY STATUS RESTARTS AGE
nfs-client-provisioner-9bf85b8fc-fcj7k 1/1 Running 0 23s
查看是否部署成功
kubectl get StorageClass
NAME PROVISIONER AGE
managed-nfs-storage fuseim.pri/ifs 61s
部署监控系统
注意需要修改的配置文件
配置configmap文件
cat >prometheus-configmap.yaml<<EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: kube-system
data:
prometheus.yml: |
global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/rules.yml
alerting:
alertmanagers:
- static_configs:
- targets: ["alertmanager:9093"]
scrape_configs:
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
- job_name: 'kubernetes-services'
kubernetes_sd_configs:
- role: service
metrics_path: /probe
params:
module: [http_2xx]
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_name
- job_name: 'kubernetes-ingresses'
kubernetes_sd_configs:
- role: ingress
relabel_configs:
- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
action: keep
regex: true
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
- job_name: 'kubernetes_node'
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
# 基于endpoint的服务发现,不再经过service代理层面
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_endpoint_port_name]
regex: true;prometheus-node-exporter
action: keep
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: (.+)(?::\d+);(\d+)
replacement: $1:$2
# 去掉label name中的前缀__meta_kubernetes_service_label_
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
# 为了区分所属node,把instance 从node-exporter ep的实例,替换成ep所在node的ip
- source_labels: [__meta_kubernetes_pod_host_ip]
regex: '(.*)'
replacement: '${1}'
target_label: instance
EOF
部署
kubectl apply -f prometheus-configmap.yaml
部署prometheus工作主程序,注意挂载上面的configmap:
cat >prometheus.deploy.yml<<EOF
apiVersion: apps/v1beta2
kind: Deployment
metadata:
labels:
name: prometheus-deployment
name: prometheus
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
containers:
- image: prom/prometheus:v2.0.0
name: prometheus
command:
- "/bin/prometheus"
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention=24h"
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- mountPath: "/prometheus"
name: data
- mountPath: "/etc/prometheus"
name: config-volume
resources:
requests:
cpu: 100m
memory: 100Mi
limits:
cpu: 500m
memory: 2500Mi
serviceAccountName: prometheus
volumes:
- name: data
emptyDir: {}
- name: config-volume
configMap:
name: prometheus-config
EOF
部署
kubectl apply -f prometheus.deploy.yml
部署svc、ingress、rbac授权。
注意:在本地是使用traefik做对外服务代理的,因此修改了默认的NodePort的svc.type为ClusterIP的方式,Ingress添加后,可以以域名方式直接访问。若不做代理,可以无需部署ingress,svc.type使用默认的NodePort,然后通过node ip+port的形式访问
cat >prometheus.svc.yaml<<EOF
kind: Service
apiVersion: v1
metadata:
labels:
app: prometheus
name: prometheus
namespace: kube-system
spec:
type: ClusterIP
ports:
- port: 80
protocol: TCP
targetPort: 9090
selector:
app: prometheus
EOF
部署
kubectl apply -f prometheus.svc.yaml
prometheus-route.yaml文件
cat >prometheus-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
name: prometheus
namespace: kube-system
spec:
entryPoints:
- web
routes:
- match: Host(`prometheus.ygbx.com`)
kind: Rule
services:
- name: prometheus
port: 80
EOF
部署
kubectl apply -f prometheus-route.yaml
rbac-setup.yam文件
cat >rbac-setup.yaml<<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: kube-system
EOF
部署
kubectl apply -f prometheus-route.yaml
配置好dns记录
访问http://prometheus.ygbx.com/随便选取一个metric,点击execute,查看是否能正常获取结果输出。点击status—target,可以看到metrics的数据来源,即各exporter,点击相应exporter上的链接可查看这个exporter提供的metrics明细。
为了更好的展示图形效果,需要部署grafana,因此前已经部署有grafana,这里不再部署,贴一个all-in-one.yaml部署文件。
cat >grafana-all-in-one.yaml<<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: grafana-core
namespace: kube-system
labels:
app: grafana
component: core
spec:
replicas: 1
template:
metadata:
labels:
app: grafana
component: core
spec:
containers:
- image: grafana/grafana:4.2.0
name: grafana-core
imagePullPolicy: IfNotPresent
# env:
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
env:
# The following env variables set up basic auth twith the default admin user and admin password.
- name: GF_AUTH_BASIC_ENABLED
value: "true"
- name: GF_AUTH_ANONYMOUS_ENABLED
value: "false"
# - name: GF_AUTH_ANONYMOUS_ORG_ROLE
# value: Admin
# does not really work, because of template variables in exported dashboards:
# - name: GF_DASHBOARDS_JSON_ENABLED
# value: "true"
readinessProbe:
httpGet:
path: /login
port: 3000
# initialDelaySeconds: 30
# timeoutSeconds: 1
volumeMounts:
- name: grafana-persistent-storage
mountPath: /var
volumes:
- name: grafana-persistent-storage
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: grafana
namespace: kube-system
labels:
app: grafana
component: core
spec:
type: NodePort
ports:
- port: 3000
selector:
app: grafana
component: core
EOF
创建ingress访问
cat >grafana-route.ymal<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
name: grafana
namespace: kube-system
spec:
entryPoints:
- web
routes:
- match: Host(`grafana.ygbx.com`)
kind: Rule
services:
- name: grafana
port: 3000
EOF
配置好dns记录
默认管理账号密码为admin admin
选择资源类型,填入prometheus的服务地址及端口号,点击保存
导入展示模板:
点击dashboard,点击import dashboard,在弹出框内填写数字315,会自动加载官方提供的315号模板,然后选择数据源为刚添加的数据源,模板就创建好了.
基本部署到这里就结束了
10容器日志收集方案
把log-agent打包至业务镜像
日志落地至物理节点
每个物理节点启动日志容器
本例中在每个node 节点部署一个pod 收集日志
安装日志组件
设置serviceAccount
kubectl create serviceaccount admin -n kube-system
配置权限
mkdir es
cat >es-rbac.yaml<<EOF
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: es-rbac
subjects:
- kind: ServiceAccount
name: admin
namespace: kube-system
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
EOF
创建权限
kubectl apply -f es-rbac.yaml
安装Elasticsearch
docker pull registry.cn-hangzhou.aliyuncs.com/cqz/elasticsearch:5.5.1
wget https://acs-logging.oss-cn-hangzhou.aliyuncs.com/elasticsearch.yml
更多推荐
所有评论(0)