从零开始搭建内网高可用k8s集群+ceph分布式存储系统+prometheus监控

前言

前段时间由于项目需求，需要搭建k8s集群，由于是首次接触，因此经验不足，只搭建了一个最基础的简单集群，现重新在内网搭建高可用集群+分布式存储，记录搭建过程

前期准备

服务器硬件准备

服务器ip系统角色
master1172.16.140.100centos7k8s-master节点1，ceph-node，ceph-osd，ceph-mds
master2172.16.140.101centos7k8s-master节点2，ceph-node，ceph-osd，ceph-mds
node1172.16.140.102centos7k8s-node节点1，ceph-node，ceph-osd，ceph-mds
node2172.16.140.103centos7k8s-node节点2，ceph-node，ceph-osd，ceph-mds，ceph-mgr，yum本地源服务器
test——外网服务器，下软件包用192.168.2.190centos7无

服务器软件准备

  • 保证master1，master2，node1，node2互通
  • 配置hosts，hostname
# 配置hostname
hostnamectl set-hostname master1
hostnamectl set-hostname master2
hostnamectl set-hostname node1
hostnamectl set-hostname node2
# 每台机器配置hosts
cat >/etc/hosts <<EOF 
172.16.140.100 master1
172.16.140.101 master2
172.16.140.102 node1
172.16.140.103 node2
EOF
  • 关闭防火墙，selinux，swap
systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
swapoff -a
sed -i 's/.*swap.*/#&/' /etc/fstab
  • 配置内核参数，iptables
# 配置k8s iptables
cat << EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF
# 配置ceph iptables
cat > /etc/sysctl.d/ceph.conf <<EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

第一步-搭建内网yum仓库

root@test：

  • 安装必要工具
yum -y install wget
yum -y install epel-release
yum -y install yum-utils
yum -y install createrepo
  • 修改本机源
# 配置国内epel源，备份原先源
mkdir /etc/yum.repos.d/bak && mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.cloud.tencent.com/repo/centos7_base.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.cloud.tencent.com/repo/epel-7.repo
# 修改国内ceph源
cat << EOF >/etc/yum.repos.d/ceph.repo
[ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/noarch/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc
[ceph-x86_64]
name=Ceph packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-nautilus/el7/x86_64/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc
EOF
# 修改国内k8s源
cat << EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 配置国内docker源
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
# 更新repo
yum clean all && yum makecache
  • 下载安装所需要的所有仓库
mkdir /root/local-repo
# 下载到本地
repotrack \
openssh \
createrepo \
yum-utils \
ceph \
ceph-mgr \
ceph-mon \
ceph-mds \
ceph-osd \
ceph-fuse \
ceph-radosgw \
ceph-mgr-dashboard \
rbd-fuse \
yum-plugin-priorities \
python-backports \
python-execnet \
python-ipaddress \
python-remoto \
python-setuptools \
ntp \
ntpdate \
ntp-doc \
docker-ce \
docker-compose \
kubelet-1.16.0-0 \
kubeadm-1.16.0-0 \
kubectl-1.16.0-0 \
haproxy \
keepalived \
net-tools \
nmap-ncat \
-p /root/local-repo
# 生成仓库
createrepo -v /root/local-repo
# 打包
tar -zcf local-repo.tar.gz local-repo
  • 将local-repo.tar.gz拷贝至node2服务器/root文件夹下

root@node2:

  • 先搭建本地源
# 解压文件
tar -xvf local-repo.tar.gz
# 备份源
mkdir /etc/yum.repos.d/bak && mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak
# 写入本地源文件
vi /etc/yum.repos.d/local-repo.repo

local-repo.repo

[localrepo]

name=Local Repository
baseurl=file:///root/local-repo
gpgcheck=0
enabled=1
  • 安装搭建仓库需要的软件
yum clean all && yum makecache
yum -y install createrepo
  • 搭建本地仓库
cd /root
# 启动http服务
nohup python -m SimpleHTTPServer 20080 &

root@master1, master2, node1

  • 配置本地仓库路径
vi /etc/yum.repos.d/local-repo.repo

[localcephrepo]

name=Local Repository
baseurl=http://172.16.140.103:20080/local-repo/
gpgcheck=0
enabled=1

第二步-搭建内网harbor服务器

root@test:

  • 安装docker，docker-compose并启动
yum -y install docker-ce docker-compose
systemctl start docker
docker version
  • 下载harbor离线包
curl -O https://github.com/goharbor/harbor/releases/download/v1.9.3/harbor-offline-installer-v1.9.3.tgz
  • 下载所需要image，打标并保存image
# 下载所需镜像
docker pull nginx:latest
docker pull quay.mirrors.ustc.edu.cn/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1
docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.16.0
docker pull kubernetesui/metrics-scraper:v1.0.1
docker pull quay.mirrors.ustc.edu.cn/coreos/flannel:v0.11.0-amd64
docker pull quay.mirrors.ustc.edu.cn/external_storage/rbd-provisioner:latest
docker pull quay.mirrors.ustc.edu.cn/external_storage/cephfs-provisioner:latest
docker pull registry.aliyuncs.com/google_containers/pause:3.1
docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.16.0
docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.16.0
docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.16.0
docker pull registry.aliyuncs.com/google_containers/etcd:3.3.15
docker pull registry.aliyuncs.com/google_containers/coredns:1.6.2
docker pull quay.mirrors.ustc.edu.cn/prometheus/node-exporter:v0.18.1
docker pull quay.mirrors.ustc.edu.cn/coreos/kube-rbac-proxy:v0.4.1
docker pull quay.mirrors.ustc.edu.cn/coreos/prometheus-operator:v0.34.0
docker pull grafana/grafana:6.4.3
docker pull quay.mirrors.ustc.edu.cn/prometheus/alertmanager:v0.18.0
docker pull kubernetesui/dashboard:v2.0.0-rc3
docker pull quay.mirrors.ustc.edu.cn/coreos/prometheus-config-reloader:v0.34.0
docker pull quay.mirrors.ustc.edu.cn/coreos/kube-state-metrics:v1.8.0
docker pull quay.mirrors.ustc.edu.cn/prometheus/prometheus:v2.11.0
docker pull quay.mirrors.ustc.edu.cn/coreos/k8s-prometheus-adapter-amd64:v0.5.0
docker pull loveone/kubernetes-dashboard-amd64:v1.10.1
docker pull quay.mirrors.ustc.edu.cn/coreos/configmap-reload:v0.0.1
docker pull registry.aliyuncs.com/google_containers/busybox:1.24
# 将镜像打标
docker tag nginx:latest 172.16.140.103/kubernetes-deploy/nginx:latest
docker tag quay.mirrors.ustc.edu.cn/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1 172.16.140.103/kubernetes-deploy/nginx-ingress-controller:0.26.1
docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.16.0 172.16.140.103/kubernetes-deploy/kube-proxy:v1.16.0
docker tag kubernetesui/metrics-scraper:v1.0.1 172.16.140.103/kubernetes-deploy/metrics-scraper:v1.0.1
docker tag quay.mirrors.ustc.edu.cn/coreos/flannel:v0.11.0-amd64 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-amd64
docker tag quay.mirrors.ustc.edu.cn/external_storage/rbd-provisioner:latest 172.16.140.103/kubernetes-deploy/rbd-provisioner:latest
docker tag quay.mirrors.ustc.edu.cn/external_storage/cephfs-provisioner:latest 172.16.140.103/kubernetes-deploy/cephfs-provisioner:latest
docker tag registry.aliyuncs.com/google_containers/pause:3.1 172.16.140.103/kubernetes-deploy/pause:3.1
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.16.0 172.16.140.103/kubernetes-deploy/kube-apiserver:v1.16.0
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.16.0 172.16.140.103/kubernetes-deploy/kube-controller-manager:v1.16.0
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.16.0 172.16.140.103/kubernetes-deploy/kube-scheduler:v1.16.0
docker tag registry.aliyuncs.com/google_containers/etcd:3.3.15 172.16.140.103/kubernetes-deploy/etcd:3.3.15
docker tag registry.aliyuncs.com/google_containers/coredns:1.6.2 172.16.140.103/kubernetes-deploy/coredns:1.6.2
docker tag quay.mirrors.ustc.edu.cn/prometheus/node-exporter:v0.18.1 172.16.140.103/kubernetes-deploy/node-exporter:v0.18.1
docker tag quay.mirrors.ustc.edu.cn/coreos/kube-rbac-proxy:v0.4.1 172.16.140.103/kubernetes-deploy/kube-rbac-proxy:v0.4.1
docker tag quay.mirrors.ustc.edu.cn/coreos/prometheus-operator:v0.34.0 172.16.140.103/kubernetes-deploy/prometheus-operator:v0.34.0
docker tag grafana/grafana:6.4.3 172.16.140.103/kubernetes-deploy/grafana:6.4.3
docker tag quay.mirrors.ustc.edu.cn/prometheus/alertmanager:v0.18.0 172.16.140.103/kubernetes-deploy/alertmanager:v0.18.0
docker tag kubernetesui/dashboard:v2.0.0-rc3 172.16.140.103/kubernetes-deploy/dashboard:v2.0.0-rc3
docker tag quay.mirrors.ustc.edu.cn/coreos/prometheus-config-reloader:v0.34.0 172.16.140.103/kubernetes-deploy/prometheus-config-reloader:v0.34.0
docker tag quay.mirrors.ustc.edu.cn/coreos/kube-state-metrics:v1.8.0 172.16.140.103/kubernetes-deploy/kube-state-metrics:v1.8.0
docker tag quay.mirrors.ustc.edu.cn/prometheus/prometheus:v2.11.0 172.16.140.103/kubernetes-deploy/prometheus:v2.11.0
docker tag quay.mirrors.ustc.edu.cn/coreos/k8s-prometheus-adapter-amd64:v0.5.0 172.16.140.103/kubernetes-deploy/k8s-prometheus-adapter-amd64:v0.5.0
docker tag loveone/kubernetes-dashboard-amd64:v1.10.1 172.16.140.103/kubernetes-deploy/kubernetes-dashboard-amd64:v1.10.1
docker tag quay.mirrors.ustc.edu.cn/coreos/configmap-reload:v0.0.1 172.16.140.103/kubernetes-deploy/configmap-reload:v0.0.1
docker tag registry.aliyuncs.com/google_containers/busybox:1.24 172.16.140.103/kubernetes-deploy/busybox:1.24
# 将镜像保存至本地
docker save 172.16.140.103/kubernetes-deploy/nginx:latest -o images/nginx.tar
docker save 172.16.140.103/kubernetes-deploy/nginx-ingress-controller:0.26.1 -o images/nginx-ingress.tar
docker save 172.16.140.103/kubernetes-deploy/kube-proxy:v1.16.0 -o images/kube-proxy.tar
docker save 172.16.140.103/kubernetes-deploy/metrics-scraper:v1.0.1 -o images/metrics-scraper.tar
docker save 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-amd64 -o images/flannel.tar
docker save 172.16.140.103/kubernetes-deploy/rbd-provisioner:latest -o images/rbd-provisioner.tar
docker save 172.16.140.103/kubernetes-deploy/cephfs-provisioner:latest -o images/cephfs-provisioner.tar
docker save 172.16.140.103/kubernetes-deploy/pause:3.1 -o images/pause.tar
docker save 172.16.140.103/kubernetes-deploy/kube-apiserver:v1.16.0 -o images/kube-apiserver.tar
docker save 172.16.140.103/kubernetes-deploy/kube-controller-manager:v1.16.0 -o images/kube-controller-manager.tar
docker save 172.16.140.103/kubernetes-deploy/kube-scheduler:v1.16.0 -o images/kube-scheduler.tar
docker save 172.16.140.103/kubernetes-deploy/etcd:3.3.15 -o images/etcd.tar
docker save 172.16.140.103/kubernetes-deploy/coredns:1.6.2 -o images/coredns.tar
docker save 172.16.140.103/kubernetes-deploy/node-exporter:v0.18.1 -o images/node-exporter.tar
docker save 172.16.140.103/kubernetes-deploy/kube-rbac-proxy:v0.4.1 -o images/kube-rbac-proxy.tar
docker save 172.16.140.103/kubernetes-deploy/prometheus-operator:v0.34.0 -o images/prometheus-operator.tar
docker save 172.16.140.103/kubernetes-deploy/grafana:6.4.3 -o images/grafana.tar
docker save 172.16.140.103/kubernetes-deploy/alertmanager:v0.18.0 -o images/alertmanager.tar
docker save 172.16.140.103/kubernetes-deploy/dashboard:v2.0.0-rc3 -o images/dashboard.tar
docker save 172.16.140.103/kubernetes-deploy/prometheus-config-reloader:v0.34.0 -o images/prometheus-config-reloader.tar
docker save 172.16.140.103/kubernetes-deploy/kube-state-metrics:v1.8.0 -o images/kube-state-metrics.tar
docker save 172.16.140.103/kubernetes-deploy/prometheus:v2.11.0 -o images/prometheus.tar
docker save 172.16.140.103/kubernetes-deploy/k8s-prometheus-adapter-amd64:v0.5.0 -o images/k8s-prometheus-adapter-amd64.tar
docker save 172.16.140.103/kubernetes-deploy/kubernetes-dashboard-amd64:v1.10.1 -o images/kubernetes-dashboard-amd64.tar
docker save 172.16.140.103/kubernetes-deploy/configmap-reload:v0.0.1 -o images/configmap-reload.tar
docker save 172.16.140.103/kubernetes-deploy/busybox:1.24 -o images/busybox.tar
  • 拷贝images文件夹至node2的/root文件夹下
  • 拷贝harbor-offline-installer-v1.9.3.tgz至node2的/root文件夹下

root@ALL:

  • 安装docker，docker-compose并设置相关参数
# docker-compose可以只在node2上安装
yum -y install docker-compose
yum -y install docker-ce
systemctl start docker
systemctl enable docker
# 配置docker启动方式为systemd和为harbor服务器设置非https链接
cat << EOF > /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "insecure-registries": ["172.16.140.103"]
}
EOF
systemctl restart docker
docker info | grep Cgroup

root@node2:

  • 解压harbor离线包
tar -xvf /root/harbor-offline-installer-v1.9.3.tgz
cd harbor/
  • 修改harbor文件夹下的文件harbor.yml
sed -i 's/reg.mydomain.com/172.16.140.103/g' harbor.yml
  • 安装harbor
./install.sh

  • 登陆harbor

  • 浏览器打开http://172.16.140.103/harbor/
    在这里插入图片描述

  • 输入用户名admin，密码Harbor12345登陆

  • 新建项目kubernetes-deploy

  • 注：harbor服务器的启动和停止：
    docker-compose start/docker-compose stop

  • docker加载镜像

docker load -i nginx.tar
docker load -i nginx-ingress.tar
docker load -i kube-proxy.tar
docker load -i metrics-scraper.tar
docker load -i flannel.tar
docker load -i rbd-provisioner.tar
docker load -i cephfs-provisioner.tar
docker load -i pause.tar
docker load -i kube-apiserver.tar
docker load -i kube-controller-manager.tar
docker load -i kube-scheduler.tar
docker load -i etcd.tar
docker load -i coredns.tar
docker load -i node-exporter.tar
docker load -i kube-rbac-proxy.tar
docker load -i dashboard.tar
docker load -i prometheus-operator.tar
docker load -i grafana.tar
docker load -i alertmanager.tar
docker load -i dashboard.tar
docker load -i prometheus-config-reloader.tar
docker load -i kube-state-metrics.tar
docker load -i prometheus.tar
docker load -i k8s-prometheus-adapter-amd64.tar
docker load -i kubernetes-dashboard-amd64.tar
docker load -i configmap-reload.tar
docker load -i busybox.tar
  • 上传至仓库
docker push 172.16.140.103/kubernetes-deploy/nginx:latest
docker push 172.16.140.103/kubernetes-deploy/nginx-ingress-controller:0.26.1
docker push 172.16.140.103/kubernetes-deploy/kube-proxy:v1.16.0
docker push 172.16.140.103/kubernetes-deploy/metrics-scraper:v1.0.1
docker push 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-amd64
docker push 172.16.140.103/kubernetes-deploy/rbd-provisioner:latest
docker push 172.16.140.103/kubernetes-deploy/cephfs-provisioner:latest
docker push 172.16.140.103/kubernetes-deploy/pause:3.1
docker push 172.16.140.103/kubernetes-deploy/kube-apiserver:v1.16.0
docker push 172.16.140.103/kubernetes-deploy/kube-controller-manager:v1.16.0
docker push 172.16.140.103/kubernetes-deploy/kube-scheduler:v1.16.0
docker push 172.16.140.103/kubernetes-deploy/etcd:3.3.15
docker push 172.16.140.103/kubernetes-deploy/coredns:1.6.2
docker push 172.16.140.103/kubernetes-deploy/node-exporter:v0.18.1
docker push 172.16.140.103/kubernetes-deploy/kube-rbac-proxy:v0.4.1
docker push 172.16.140.103/kubernetes-deploy/prometheus-operator:v0.34.0
docker push 172.16.140.103/kubernetes-deploy/grafana:6.4.3
docker push 172.16.140.103/kubernetes-deploy/alertmanager:v0.18.0
docker push 172.16.140.103/kubernetes-deploy/dashboard:v2.0.0-rc3
docker push 172.16.140.103/kubernetes-deploy/prometheus-config-reloader:v0.34.0
docker push 172.16.140.103/kubernetes-deploy/kube-state-metrics:v1.8.0
docker push 172.16.140.103/kubernetes-deploy/prometheus:v2.11.0
docker push 172.16.140.103/kubernetes-deploy/k8s-prometheus-adapter-amd64:v0.5.0
docker push 172.16.140.103/kubernetes-deploy/kubernetes-dashboard-amd64:v1.10.1
docker push 172.16.140.103/kubernetes-deploy/configmap-reload:v0.0.1
docker push 172.16.140.103/kubernetes-deploy/busybox:1.24
  • 在网页上查看仓库是否已经保存镜像
    在这里插入图片描述

第三步-在master1，master2节点上搭建haproxy+keepalived框架

root@master1，master2:

  • 安装相关插件
yum install -y haproxy keepalived net-tools nmap-ncat
  • 配置haproxy
sed -i '$a\
#---------------------------------------------------------------------\
# kube-api-server-listen\
#---------------------------------------------------------------------\
listen kube-api-lb\
	bind 0.0.0.0:10443\
	mode tcp\
	balance roundrobin\
	server master1 172.16.140.100:6443 weight 1 maxconn 10000 check inter 10s\
	server master2 172.16.140.101:6443 weight 1 maxconn 10000 check inter 10s\
\
#---------------------------------------------------------------------\
# kube-api-haproxy-stats\
#---------------------------------------------------------------------\
listen admin_stats\
	bind 0.0.0.0:8099\
	mode http\
	option httplog\
	maxconn 10\
	stats refresh 30s\
	stats uri /stats' /etc/haproxy/haproxy.cfg
  • 配置keepalived
mkdir -p /etc/keepalived/scripts/
cat > /etc/keepalived/scripts/haproxy_check.sh << \EOF
#!/bin/bash
if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]
then
	systemctl start haproxy
	sleep 3
	if [ `ps -C haproxy --no-header |wc -l` -eq 0 ]
	then
		systemctl stop keepalived
	fi
fi
EOF
cat > /etc/keepalived/scripts/notifi_master.sh << \EOF
#!/bin/bash
VIP=172.xx.xx.xx
GATEWAY=172.xx.xx.xx
/sbin/arping -I eth0 -c 5 -s $VIP $GATEWAY &>/dev/null
EOF
chmod +x /etc/keepalived/scripts/haproxy_check.sh /etc/keepalived/scripts/notifi_master.sh
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.backup
  • 这里把master1节点作为主节点，master2节点作为备用节点
    在master1节点上配置主keepalived.conf
cat << EOF > /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id Haproxy-Master
   script_user root
   enable_script_security
   vrrp_skip_check_adv_addr
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
   # vrrp_strict
}

vrrp_script chk_haproxy
{
    script "/etc/keepalived/scripts/haproxy_check.sh"
    interval 5
    fall 2
}

vrrp_instance haproxy {
    state MASTER
    interface ens32
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS      
        auth_pass 5e97s45a2  
    }
    unicast_src_ip 172.16.140.100
    unicast_peer {
      172.16.140.101
      172.16.140.102
      172.16.140.103
    }
	track_script {
        chk_haproxy
    }
    virtual_ipaddress {
        172.16.140.150
    }
	notify_master "/etc/keepalived/scripts/notifi_master.sh"
}

EOF

在master2节点上配置备keepalived.conf

cat << EOF > /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   router_id Haproxy-Master
   script_user root
   enable_script_security
   vrrp_skip_check_adv_addr
   vrrp_iptables
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script chk_haproxy
{
    script "/etc/keepalived/scripts/haproxy_check.sh"
    interval 5
    fall 2
}

vrrp_instance haproxy {
    state BACKUP
    interface ens32
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS      
        auth_pass 5e97s45a2  
    }
    unicast_src_ip 172.16.140.101
    unicast_peer {
      172.16.140.100
      172.16.140.102
      172.16.140.103
    }
    virtual_ipaddress {
        172.16.140.150
    }
	notify_master "/etc/keepalived/scripts/notifi_master.sh"
}

EOF
  • 启动keepalived
systemctl stop NetworkManager
systemctl start keepalived
systemctl enable keepalived
netstat -ntplu|grep 10443

第四步-部署k8s集群

root@ALL:

  • 安装kubelet，kubectl，kubeadm
yum install -y kubelet-1.16.0 kubeadm-1.16.0 kubectl-1.16.0
systemctl enable kubelet

root@master1：

  • 使用kubeadm部署master1
    pod-network-cidr为flanneld网络默认地址
kubeadm init --kubernetes-version=1.16.0 \
--apiserver-advertise-address=172.16.140.100 \
--image-repository 172.16.140.103/kubernetes-deploy \
--service-cidr=10.1.0.0/16 \
--control-plane-endpoint "172.16.140.150:10443" \
--upload-certs \
--pod-network-cidr=10.244.0.0/16
  • 记录加入master节点的token和加入node节点的token
  • 配置kubectl工具
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get nodes
kubectl get cs
  • 部署flanneld网络
    创建文件flanneld.yaml
# 生成文件夹
mkdir -p yaml/flanneld
# 写入文件
cat << EOF > yaml/flanneld/flanneld.yaml 
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
    - configMap
    - secret
    - emptyDir
    - hostPath
  allowedHostPaths:
    - pathPrefix: "/etc/cni/net.d"
    - pathPrefix: "/etc/kube-flannel"
    - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups: ['extensions']
    resources: ['podsecuritypolicies']
    verbs: ['use']
    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-amd64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - amd64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-amd64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - arm64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-arm64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-arm64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - arm
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-arm
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-arm
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-ppc64le
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - ppc64le
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-ppc64le
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-ppc64le
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-s390x
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - s390x
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-s390x
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: 172.16.140.103/kubernetes-deploy/flannel:v0.11.0-s390x
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
EOF
  • 创建flanneld网络
kubectl create -f yaml/flanneld/flanneld.yaml
# 检测集群状态
kubectl get nodes

root@master2：

  • 使用之前记录的加入命令加入k8s集群（control-plane）

root@node1，node2：

  • 使用之前记录的加入命令加入k8s集群
  • 在master1或者master2节点上查看集群状态
kubectl get pod,svc,cs,node -Ao wide

第五步-部署ingress

root@master1

  • 生成nginx-ingress部署文件
# 创建目录
mkdir yaml/nginx-ingress
# 生成文件
cat << EOF > yaml/nginx-ingress/deployment.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
      - "networking.k8s.io"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      # wait up to five minutes for the drain of connections
      terminationGracePeriodSeconds: 300
      serviceAccountName: nginx-ingress-serviceaccount
      nodeSelector:
        kubernetes.io/os: linux
      containers:
        - name: nginx-ingress-controller
          image: 172.16.140.103/kubernetes-deploy/nginx-ingress-controller:0.26.1
          args:
            - /nginx-ingress-controller
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
            - name: https
              containerPort: 443
              protocol: TCP
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          lifecycle:
            preStop:
              exec:
                command:
                  - /wait-shutdown

---

apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
      nodePort: 30080
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
      nodePort: 30443
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
EOF
# 部署nginx-ingress
kubectl apply -f yaml/nginx-ingress/deployment.yaml

第五步-部署dashboard

root@master1:

  • 生成dashboard.yaml
# 生成文件夹
mkdir yaml/kube-dashboard
# 创建文件
cat << \EOF > yaml/kube-dashboard/deploy.yaml
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: Namespace
metadata:
  name: kubernetes-dashboard

---

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kubernetes-dashboard
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kubernetes-dashboard
type: Opaque
data:
  csrf: ""

---

apiVersion: v1
kind: Secret
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kubernetes-dashboard
type: Opaque

---

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kubernetes-dashboard

---

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
rules:
  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs: ["get", "update", "delete"]
    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["kubernetes-dashboard-settings"]
    verbs: ["get", "update"]
    # Allow Dashboard to get metrics.
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["heapster", "dashboard-metrics-scraper"]
    verbs: ["proxy"]
  - apiGroups: [""]
    resources: ["services/proxy"]
    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs: ["get"]

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
rules:
  # Allow Metrics Scraper to get metrics from the Metrics server
  - apiGroups: ["metrics.k8s.io"]
    resources: ["pods", "nodes"]
    verbs: ["get", "list", "watch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubernetes-dashboard
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kubernetes-dashboard
subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kubernetes-dashboard

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      containers:
        - name: kubernetes-dashboard
          image: 172.16.140.103/kubernetes-deploy/dashboard:v2.0.0-rc3
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --namespace=kubernetes-dashboard
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "beta.kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule

---

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper

---

kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kubernetes-dashboard
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
      annotations:
        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
    spec:
      containers:
        - name: dashboard-metrics-scraper
          image: 172.16.140.103/kubernetes-deploy/metrics-scraper:v1.0.1
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "beta.kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      volumes:
        - name: tmp-volume
          emptyDir: {}
EOF
  • 生成dahsboard
kubectl apply -f yaml/kube-dashboard/deploy.yaml
  • 替换过期证书
#生成证书
mkdir -p key/kube-dashboard-key
openssl req -new -nodes -x509 \
  -subj "/O=IT/CN=kube-dashboard.com" -days 3650 \
  -keyout key/kube-dashboard-key/tls.key \
  -out key/kube-dashboard-key/tls.crt \
  -extensions v3_ca
#删除原有的证书secret
kubectl delete secret kubernetes-dashboard-certs -n kubernetes-dashboard
#创建新的证书secret
kubectl create secret tls kubernetes-dashboard-certs --cert='key/kube-dashboard-key/tls.crt' --key='key/kube-dashboard-key/tls.key' -n kubernetes-dashboard
#重启pod
kubectl delete pod $(kubectl -n kubernetes-dashboard get pod | awk '/kubernetes-dashboard/{print $1}') -n kubernetes-dashboard
  • 配置ingress
# 生成ingress
openssl req -new -nodes -x509 \
  -subj "/O=IT/CN=kubernetes-dashboard.com" -days 3650 \
  -keyout key/nginx-ingress/kubernetes-dashboard/tls.key \
  -out key/nginx-ingress/kubernetes-dashboard/tls.crt \
  -extensions v3_ca
# 新建ingress的tls-secret
kubectl create secret tls ingress-kubernetes-dashboard-certs --cert='key/nginx-ingress/kubernetes-dashboard/tls.crt' --key='key/nginx-ingress/kubernetes-dashboard/tls.key' -n kubernetes-dashboard
# 生成ingress规则
cat << EOF > yaml/nginx-ingress/kubernetes-dashboard-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-kubernetes-dashboard-tls
  namespace: kubernetes-dashboard
  annotations:
    nginx.ingress.kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
    nginx.ingress.kubernetes.io/rewrite-target: /\$1
spec:
  tls:
   - secretName: ingress-kubernetes-dashboard-certs
  rules:
   - http:
      paths:
      - path: /dashboard/(.*)
        backend:
          serviceName: kubernetes-dashboard
          servicePort: 443
EOF
# 部署ingress规则
kubectl apply -f yaml/nginx-ingress/kubernetes-dashboard-ingress.yaml

这里遇到两个个大坑，一天半时间才解决，一个是由于dashboard本身为https访问，按照网上说的配置后端的访问需要设置ingress的选项为：

nginx.ingress.kubernetes.io/secure-backends: "true"

但是这个已经在官方文档被废弃了，实际应该为：

nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"

第二个是关于重定向地址的填写，原来写的是：

nginx.ingress.kubernetes.io/rewrite-target: /

但是这个会导致dashboard匹配出错，因为dashboard重定向之后会再进行一次重定向，导致匹配规则失败，因此只能用正则来进行匹配，才能完美重定向，因此应该这么写：

nginx.ingress.kubernetes.io/rewrite-target: /$1

这两个坑让我学到还是要多看官方文档，不能依赖网络上的说法

  • 创建账号访问
kubectl create serviceaccount  dashboard-admin -n kubernetes-dashboard
kubectl create clusterrolebinding  dashboard-admin --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:dashboard-admin
# 获取账号token
kubectl describe secrets -n kubernetes-dashboard $(kubectl -n kubernetes-dashboard get secret | awk '/dashboard-admin/{print $1}')
  • 登陆dashboard
    输入https://172.16.140.150:30443/dashboard/访问dashboard
    在这里插入图片描述

第六步-搭建ceph

root@ALL:

  • 配置myceph用户
useradd -d /home/myceph -m myceph
passwd  myceph
  • 为myceph添加sudo权限
sed -i "/## Allow root to run any commands anywhere /a\\\myceph ALL = (ALL)  ALL" /etc/sudoers
echo "myceph ALL = (root) NOPASSWD:ALL" | tee /etc/sudoers.d/myceph
sudo chmod 0440 /etc/sudoers.d/myceph
  • 在所有节点安装ceph组件
yum install -y ceph ceph-radosgw yum-plugin-priorities
  • 安装ntp服务
yum install -y ntp ntpdate ntp-doc
  • 配置ntp内网同步
    这里以node2作为ntp服务器
    在node2上修改ntp.conf
vi /etc/ntp.conf

ntp.conf

# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
restrict 127.0.0.1 
restrict ::1

# Hosts on local network are less restricted.
#restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server 127.127.1.0
fudge 127.127.1.0 stratum 10
restrict 172.16.140.0 mask 255.255.255.0 #允许客户端网段的所有主机向本机请求时间同步

#broadcast 192.168.1.255 autokey	# broadcast server
#broadcastclient			# broadcast client
#broadcast 224.0.1.1 autokey		# multicast server
#multicastclient 224.0.1.1		# multicast client
#manycastserver 239.255.254.254		# manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography. 
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats

# Disable the monitoring facility to prevent amplification attacks using ntpdc
# monlist command when default restrict does not include the noquery flag. See
# CVE-2013-5211 for more details.
# Note: Monitoring will not be disabled with the limited restriction flag.
disable monitor

主要在serverpool下方新增ntp本地server并注释原先互联网ntp-server
server 127.127.1.0
fudge 127.127.1.0 stratum 10
restrict 172.16.140.0 mask 255.255.255.0

  • 在root@node1，naster1，master2上设置ntp同步
    修改ntp.conf
# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
restrict 127.0.0.1 
restrict ::1

# Hosts on local network are less restricted.
#restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server 172.16.140.103    #指名上层NTP服务器
restrict 172.16.140.103       #放行

#broadcast 192.168.1.255 autokey	# broadcast server
#broadcastclient			# broadcast client
#broadcast 224.0.1.1 autokey		# multicast server
#multicastclient 224.0.1.1		# multicast client
#manycastserver 239.255.254.254		# manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography. 
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats

# Disable the monitoring facility to prevent amplification attacks using ntpdc
# monlist command when default restrict does not include the noquery flag. See
# CVE-2013-5211 for more details.
# Note: Monitoring will not be disabled with the limited restriction flag.
disable monitor

设置serverpool上层服务器
server 172.16.140.103
restrict 172.16.140.103

  • 开启ntpd, 查看ntp服务是否已经启动并已经同步
# 开启ntpd
systemctl start ntpd
systemctl enable ntpd
# 查看ntpstat
ntpstat
# 查看ntpd服务器状态
ntpq -p

myceph@node2:

  • 生成rsa-key实现免密登陆
# 生成key
ssh-keygen  -t  rsa
# 拷贝到各个节点
ssh-copy-id -i .ssh/id_rsa.pub myceph@node1
ssh-copy-id -i .ssh/id_rsa.pub myceph@node2
ssh-copy-id -i .ssh/id_rsa.pub myceph@master1
ssh-copy-id -i .ssh/id_rsa.pub myceph@master2
  • 配置ssh-config
cat > ~/.ssh/config <<EOF
Host ceph-node1
Hostname master1
User myceph

Host ceph-node2
Hostname master2
User myceph

Host ceph-node3
Hostname node1
User myceph

Host ceph-node3
Hostname node2
User myceph
EOF
# 配置该文件权限
sudo chmod 600 ~/.ssh/config
  • 安装ceph-deploy
sudo yum install -y python-backports python-execnet python-ipaddress python-remoto python-setuptools
sudo yum install ceph-deploy -y
  • 搭建ceph集群
mkdir my-cluster  
cd my-cluster
# 部署mon
ceph-deploy new node2 node1
  • 修改ceph.conf在最后一行添加public network配置
vi ceph.conf

ceph.conf

[global]
fsid = f9cf1256-b100-4620-88df-2f9481fc24bd
mon_initial_members = node2, node1
mon_host = 172.16.140.103,172.16.140.102
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network = 172.16.140.0/24
  • 初始化mon，mgr，osd，mds
# 部署mon
ceph-deploy mon create-initial
# 赋予节点admin权限
ceph-deploy admin node1 node2 master1 master2
# 增加mgr
ceph-deploy mgr create node2 node1 master2 master1
# 增加osd
ceph-deploy osd create --data /dev/sdb node1
ceph-deploy osd create --data /dev/sdb node2
ceph-deploy osd create --data /dev/sdb master1
ceph-deploy osd create --data /dev/sdb master2
# 增加mds
ceph-deploy mds create node2 node1 master2 master1

root@node2:

  • 开启dashboard
yum install -y ceph-mgr-dashboard
ceph mgr module enable dashboard
# 生成证书
mkdir -p key/ceph-dashboard
openssl req -new -nodes -x509 \
  -subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 \
  -keyout key/ceph-dashboard/tls.key -out key/ceph-dashboard/tls.crt -extensions v3_ca
ceph dashboard set-ssl-certificate -i key/ceph-dashboard/tls.crt
ceph dashboard set-ssl-certificate-key -i key/ceph-dashboard/tls.key
# 设置端口，ip
ceph config set mgr mgr/dashboard/server_addr 172.16.140.103
ceph config set mgr mgr/dashboard/server_port 7000
ceph config set mgr mgr/dashboard/ssl_server_port 8443
# 设置用户
ceph dashboard ac-user-create admin admin administrator
# 查看地址
ceph mgr services
  • 登陆ceph-dashboard
    输入https://172.16.140.103:8443/ 用户admin 密码admin登陆
    在这里插入图片描述

第七步-搭建cephFS文件系统并以storageclass方式提供服务

root@node2:

  • 创建cephfs
# 创建cephfs所需pool
ceph osd pool create fs_kube_data 64
ceph osd pool create fs_kube_metadata 64
ceph fs new cephfs fs_kube_metadata fs_kube_data
  • 测试挂载cephfs
yum -y install ceph-fuse
mkdir /mnt/cephfs
# 将cephfs挂载在mnt目录下
ceph-fuse -m 172.16.140.103:6789 /mnt/cephfs
# 查看是否挂载成功
df -h |grep ceph-fuse

root@master1

  • 创建admin-secret
ceph auth get-key client.admin > /tmp/secret
kubectl create ns ceph
kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=ceph
  • 创建cephfs-provisioner文件
# 创建文件夹
mkdir -p yaml/cephfs
# 创建文件
cat << EOF > yaml/cephfs/provisioner.yaml
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "create", "delete"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cephfs-provisioner
subjects:
  - kind: ServiceAccount
    name: cephfs-provisioner
    namespace: ceph
roleRef:
  kind: ClusterRole
  name: cephfs-provisioner
  apiGroup: rbac.authorization.k8s.io

---

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: cephfs-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["create", "get", "delete"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: cephfs-provisioner
  namespace: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cephfs-provisioner
subjects:
- kind: ServiceAccount
  name: cephfs-provisioner

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: cephfs-provisioner
  namespace: ceph

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cephfs-provisioner
  namespace: ceph
spec:
  replicas: 1
  selector:
    matchLabels:
      app: cephfs-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: cephfs-provisioner
    spec:
      containers:
      - name: cephfs-provisioner
        image: "172.16.140.103/kubernetes-deploy/cephfs-provisioner:latest"
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/cephfs
        - name: PROVISIONER_SECRET_NAMESPACE
          value: ceph
        command:
        - "/usr/local/bin/cephfs-provisioner"
        args:
        - "-id=cephfs-provisioner-1"
      serviceAccount: cephfs-provisioner

---
EOF
  • 创建cephfs-provisioner
kubectl create -f yaml/cephfs/provisioner.yaml
# 查看是否创建成功
kubectl get pod -n=ceph -o wide
  • 创建storageclass文件及测试文件
# 创建storageclass文件
cat << EOF > yaml/cephfs/storageclass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: cephfs
provisioner: ceph.com/cephfs
parameters:
    monitors: 172.16.140.100:6789,172.16.140.101:6789,172.16.140.102:6789,172.16.140.103:6789
    adminId: admin
    adminSecretName: ceph-secret-admin
    adminSecretNamespace: "ceph"
    claimRoot: /pvc-volumes
allowVolumeExpansion: true
EOF
# 创建pvc测试文件
cat << EOF > yaml/cephfs/test-pvc.yaml
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-pvc
  namespace: ceph
spec:
  storageClassName: cephfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 5Gi
EOF
# 创建pod测试挂载pvc
cat << EOF > yaml/cephfs/test-pod.yaml
---

kind: Pod
apiVersion: v1
metadata:
  name: test-pod
  namespace: ceph
spec:
  containers:
  - name: test-pod
    image: 172.16.140.103/kubernetes-deploy/busybox:1.24
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1"
    volumeMounts:
      - name: pvc
        mountPath: "/mnt"
        subPath: "test-pod"
  restartPolicy: "Never"
  volumes:
    - name: pvc
      persistentVolumeClaim:
        claimName: test-pvc
EOF
  • 部署storageclass以及测试pvc，pod，查看是否成功
kubectl apply -f yaml/cephfs/storageclass.yaml
kubectl apply -f yaml/cephfs/test-pvc.yaml
# 等待创建完成，再进行测试
kubectl apply -f yaml/cephfs/test-pod.yaml	
# 查看是否创建成功
kubectl describe pvc -n=ceph
kubectl describe pod/test-pod -n=ceph

第八步-搭建cephRBD存储并以storageclass方式提供服务

root@node2:

  • 创建ceph-pool并启用mon
# 创建pool
ceph osd pool create rbd_kube 8 8
# 启用mon
ceph osd pool application enable rbd_kube mon
# 设置读写权限
ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=rbd_kube'

root@master1:

  • 创建ceph-secret
ceph auth get-key client.kube > /tmp/kube-secret
kubectl create secret generic ceph-rbd-secret --from-file=/tmp/kube-secret --namespace=ceph --type=kubernetes.io/rbd
  • 创建ceph-rbd-provisioner文件
# 创建文件夹
mkdir -p yaml/ceph-rbd
cat << EOF > yaml/ceph-rbd/provisioner.yaml
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rbd-provisioner
  namespace: ceph
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["kube-dns","coredns"]
    verbs: ["list", "get"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "create", "delete"]
    
---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rbd-provisioner
subjects:
  - kind: ServiceAccount
    name: rbd-provisioner
    namespace: ceph
roleRef:
  kind: ClusterRole
  name: rbd-provisioner
  apiGroup: rbac.authorization.k8s.io

---

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rbd-provisioner
  namespace: ceph
rules:
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]
  
---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: rbd-provisioner
  namespace: ceph
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rbd-provisioner
subjects:
- kind: ServiceAccount
  name: rbd-provisioner
  namespace: default

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: rbd-provisioner
  namespace: ceph

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: rbd-provisioner
  namespace: ceph
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rbd-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: rbd-provisioner
    spec:
      containers:
      - name: rbd-provisioner
        image: "172.16.140.103/kubernetes-deploy/rbd-provisioner:latest"
        env:
        - name: PROVISIONER_NAME
          value: ceph.com/rbd
      serviceAccount: rbd-provisioner

---
EOF
  • 部署rbd-provisioner
kubectl create -f yaml/ceph-rbd/provisioner.yaml
# 查看是否已经成功部署
kubectl get pod -n=ceph -o wide
  • 创建storageclass文件和测试文件
cat << EOF > yaml/ceph-rbd/storageclass.yaml
---

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: rbd
provisioner: ceph.com/rbd
parameters:
  monitors: 172.16.140.100:6789,172.16.140.101:6789,172.16.140.102:6789,172.16.140.103:6789
  pool: rbd_kube
  adminId: admin
  adminSecretNamespace: ceph
  adminSecretName: ceph-secret-admin
  userId: kube
  userSecretNamespace: ceph
  userSecretName: ceph-rbd-secret
  imageFormat: "2"
  imageFeatures: layering
allowVolumeExpansion: true
---
EOF
# 创建测试pvc
cat << EOF > yaml/ceph-rbd/test-pvc.yaml
---

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-pvc-rbd
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: rbd
  resources:
    requests:
      storage: 5Gi

---
EOF
# 创建测试pod
cat << EOF > yaml/ceph-rbd/test-pod.yaml
---

kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: 172.16.140.103/kubernetes-deploy/busybox:1.24
    command:
    - "/bin/sh"
    args:
    - "-c"
    - "touch /mnt/SUCCESS && exit 0 || exit 1"
    volumeMounts:
    - name: pvc
      mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
  - name: pvc
    persistentVolumeClaim:
      claimName: test-pvc-rbd

---
EOF
  • 测试pod挂载pvc（RBD方式）
kubectl create -f yaml/ceph-rbd/storageclass.yaml
kubectl create -f yaml/ceph-rbd/test-pvc.yaml
# 等前面创建完成再创建
kubectl create -f yaml/ceph-rbd/test-pod.yaml
# 查看是否创建成功
kubectl describe pvc -n=ceph
kubectl describe pod/test-pod -n=ceph

第九步-部署prometheus

root@test：

  • 下载prometheus源码
# 安装git
yum install -y git
# 下载prometheus
git clone https://github.com/coreos/kube-prometheus.git
cd kube-prometheus/manifests
# 替换prometheus镜像内容
sed -r -i "s/quay.io\/coreos/172.16.140.103\/kubernetes-deploy/g" setup/*.yaml
sed -r -i "s/quay.io\/coreos/172.16.140.103\/kubernetes-deploy/g" *.yaml
sed -r -i "s/quay.io\/prometheus/172.16.140.103\/kubernetes-deploy/g" *.yaml
sed -r -i "s/image: grafana/image: 172.16.140.103\/kubernetes-deploy/g" *.yaml
cd ../ & tar -zcf manifests.tar.gz manifests
  • 拷贝manifests.tar.gz至master1的root文件夹下

root@master1

  • 解压并安装prometheus
# 解压文件
tar -xvf manifests.tar.gz
# 安装prometheus
kubectl apply -f manifests/setup
kubectl apply -f manifests

至此，集群的搭建基本完成。

# kubernetes # ceph # docker # 分布式 # centos
Logo
K8S/Kubernetes

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐

【深度】阿里巴巴万级规模 K8s 集群全局高可用体系之美

作者 |  韩堂、柘远、沉醉来源 | 阿里巴巴云原生公众号​前言台湾作家林清玄在接受记者采访的时候，如此评价自己 30 多年写作生涯：“第一个十年我才华横溢，‘贼光闪现’，令周边黯然失色；第二个十年，我终于‘宝光现形’，不再去抢风头，反而与身边的美丽相得益彰；进入第三个十年，繁华落尽见真醇，我进入了‘醇光初现’的阶段，真正体味到了境界之美”。​长夜有穷，真水无香。领略过了 K8s“身在江

avatar K8S/Kubernetes

如何基于 K8s 构建下一代 DevOps 平台？

作者 | 孙健波（天元）导读：当前云原生 DevOps 体系现状如何？面临哪些挑战？如何通过 OAM 解决云原生 DevOps 场景下的诸多问题？云原生开发应用模型 OAM(Open Application Model) 社区核心成员孙健波将为大家一一解答，并分享如何基于 OAM 和 Kubernetes 打造无限能力的下一代 DevOps 平台。什么是 DevOps？为什么基于 Kub

avatar K8S/Kubernetes

k8s 火了！

2020，上云之年，产品云端化成为一种趋势。在一线城市，很多公司都已经构建了自己的私有云环境，比如阿里云、网易云、华为云等。而Kubernetes 作为基于容器编排领域的王者，具备扩展...

avatar K8S/Kubernetes