一、准备基础环境
cat >> /etc/hosts <<EOF
192.168.180.210 k8s-master
192.168.180.200 k8s-node1
192.168.180.190 k8s-node2
192.168.180.180 gitlab
192.168.180.170 jenkins
192.168.180.160 harbor
EOF
配置主机名
hostnamectl set-hostname k8s-master && bash
hostnamectl set-hostname k8s-node1 && bash
hostnamectl set-hostname k8s-node2 && bash
hostnamectl set-hostname gitlab && bash
hostnamectl set-hostname jenkins && bash
hostnamectl set-hostname harbor && bash
安装常用软件
yum -y install vim wget net-tools lrzsz
systemctl stop firewalld && systemctl disable firewalld
sed -i ‘/^SELINUX=/s/enforcing/disabled/’ /etc/selinux/config && setenforce 0

二、部署K8S群集
基础配置[三台centos]
1、关闭交换分区
swapoff -a && sed -ri ‘s/.swap./#&/’ /etc/fstab

2、加载模块并添加v4流量传递
modprobe br_netfilter
cat > /etc/sysctl.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl -p

3、安装ipvs
yum install -y conntrack ntpdate ntp ipvsadm ipset iptables curl sysstat libseccomp wget vim net-tools git
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe – ip_vs
modprobe – ip_vs_rr
modprobe – ip_vs_wrr
modprobe – ip_vs_sh
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
lsmod | grep -e ip_vs -e nf_conntrack

4、安装containerd
cat << EOF > /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y containerd.io docker-ce docker-ce-cli
mkdir /etc/containerd -p
containerd config default > /etc/containerd/config.toml
vim /etc/containerd/config.toml
SystemdCgroup = false 改为 SystemdCgroup = true

sandbox_image = “k8s.gcr.io/pause:3.6”

改为：
sandbox_image = “registry.aliyuncs.com/google_containers/pause:3.6”

在[plugins.“io.containerd.grpc.v1.cri”.registry.mirrors]下添加如下内容

    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.180.160"]
    endpoint = ["http://192.168.180.160"]

    [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.180.160".auth]
    username = "jenkins"
    password = "ABCabc-123"

systemctl enable containerd && systemctl start containerd

ctr version
runc -version

5、安装k8s[三台centos]
cat < /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
EOF

yum clean all
yum makecache fast
yum install -y kubectl kubelet kubeadm
systemctl enable kubelet
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS=“–cgroup-driver=systemd”

6、初始化群集[master]
kubeadm config print init-defaults > init-config.yaml
vim init-config.yaml

advertiseAddress: 192.168.180.210
name: k8s-master

dataDir: /var/lib/etcd

imageRepository: registry.aliyuncs.com/google_containers

podSubnet: 10.244.0.0/16

kubeadm init --config=init-config.yaml
export KUBECONFIG=/etc/kubernetes/admin.conf
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id-u):$(id -g) $HOME/.kube/config

加入群集[这里的token和discovery-token都是初始化群集最好所给的]
kubeadm join 192.168.10.210:6443 --token 8zgrg1.dwy5s6rqzzhlkkdl --discovery-token-ca-cert-hash sha256:9dfa30a7a8314887ea01b05cc26e80856bfd253d1a71de7cd5501c42f11c0326

部署calico网络[master]
wget https://docs.projectcalico.org/v3.18/manifests/calico.yaml --no-check-certificate
vim calico.yaml //3673行修改为如下

• name: CALICO_IPV4POOL_CIDR
  value: “10.244.0.0/16”

kubectl apply -f calico.yaml
kubectl describe node k8s-master
kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule-
kubectl get pod -n kube-system
kubectl get node

三/部署GitLab
安装GitLab
yum -y install policycoreutils-python
上传软件包
rpm -ivh gitlab-ce-12.9.0-ce.0.el7.x86_64.rpm
vim /etc/gitlab/gitlab.rb
external_url ‘http://192.168.180.180’ —修改为本机地址
gitlab-ctl reconfigure

部署好后，根据PDF的要求，上传项目文件

四、部署Jenkins
yum install -y git
安装TOMCAT
tar zxf apache-tomcat-9.0.52.tar.gz -C /usr/local
mv /usr/local/apache-tomcat-9.0.52 /usr/local/tomcat
安装JDK
tar zxf jdk-8u301-linux-x64.tar.gz
mv jdk1.8.0_301 /usr/local/java
vim /etc/profile —追加
export JAVA_HOME=/usr/local/java
export PATH=$JAV{A}_{H}OME/bin:$PATH
export CLASSPATH=.:$JAV{A}_{H}OME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
source /etc/profile

将Jenkins.war包上传到Jenkins主机的/usr/local/tomcat/webapp目录下
cd /usr/local/tomcat/webapps
上传Jenkins
启动JenKins
cd /usr/local/tomcat/bin
sh startup.sh

初始化Jenkins
cat /root/.jenkins/secrets/initialAdminPassword
Jenkins初始化配置完成后，重启下tomcat，在安装目录下先，./shutdown.sh 再 ./startup.sh
安装如下插件
CloudBees Docker Build and Publish
Git Parameter
Git plugin
创建凭据
添加登录gitlab的凭据 root ABCabc-123
添加登录harbor的凭据 jenkins ABCabc-123

安装Docker(jenkins服务器)
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-ce git
mkdir /etc/docker
cat >> /etc/docker/daemon.json <<EOF
{
“registry-mirrors”: [“https://g6yrjrwf.mirror.aliyuncs.com”],
“exec-opts”: [“native.cgroupdriver=systemd”],
“insecure-registries”: [“192.168.180.160”]
}
EOF
systemctl start docker && systemctl enable docker
cat << EOF >> /etc/sysctl.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl -p

五、部署Harbor
安装Docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-ce
mkdir /etc/docker
cat >> /etc/docker/daemon.json <<EOF
{
“registry-mirrors”: [“https://g6yrjrwf.mirror.aliyuncs.com”],
“exec-opts”: [“native.cgroupdriver=systemd”],
“insecure-registries”: [“192.168.180.160”]
}
EOF
systemctl start docker && systemctl enable docker
cat << EOF >> /etc/sysctl.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl -p

上传Docker compose，赋予执行权限，并移动到/usr/local/bin
cd /usr/local/bin
chmod +x /usr/local/bin/docker-compose

安装Harbor
上传软件包并解压
tar zxf harbor-offline-installer-v2.3.1.tgz -C /usr/local/
修改Harbor的配置文件
cd /usr/local/harbor
/bin/cp harbor.yml.tmpl harbor.yml
vim /usr/local/harbor/harbor.yml
1）将hostname修改为本机地址
hostname: 192.168.180.160
2）将https相关的配置都注释掉

启动Harbor程序
sh /usr/local/harbor/install.sh

启动Harbor程序 docker-compose up -d
关闭Harbor程序 docker-compose stop
查看Harbor程序 docker-compose ps

登录Harbor，默认用户名 admin 密码：Harbor12345
创建新用户，和jenkins的凭证相关
jenkins ABCabc-123

创建项目，将jenkins用户添加到项目中

修改Jenkins服务器上的Docker配置文件
cat /etc/docker/daemon.json
“insecure-registries”: [“192.168.180.160”] //添加，IP为Harbor的地址

六、发布应用
1、创建K8S的yaml文
在k8s-master上执行:
kubectl create secret docker-registry harbor-login-registry --docker-server=192.168.180.160 --docker-username=jenkins --docker-password=ABCabc-123
创建yaml文件
mkdir /data
cd /data
vim nginx-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
imagePullSecrets:
- name: harbor-login-registry
containers:
- name: nginx
image: 192.168.180.160/jenkins/nginx:latest
imagePullPolicy: Always
ports:
- containerPort: 80

vim nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: svc-nginx
labels:
app: svc-nginx
spec:
type: NodePort
ports:

• port: 80
  targetPort: 80
  NodePort: 30088
  selector:
  app: nginx

编写发布到K8S脚本
在k8s-master的/data目录下创建
vim deploy.sh
#!/bin/bash
cd /data
newversion=$1oldversion=$(cat nginx-deploy.yaml | grep “image:” | awk -F ‘:’ ‘{print KaTeX parse error: Expected 'EOF', got '}' at position 3: NF}̲') echo "old ve…{oldversion}
echo “new version is:”KaTeX parse error: Expected 'EOF', got '#' at position 14: {newversion} #̲replace new ver…{oldversion}"’/‘“nginx:${newversion}”’/g’ nginx-deploy.yaml
#deploy
kubectl delete -f nginx-deploy.yaml
kubectl create -f nginx-deploy.yaml

｛宿主机（相当于用户电脑）上操作｝
上传Dockerfile及nginx.repo到GitLab
在宿主机上创建Dockerfile、nginx.repo
FROM centos:7
LABEL maintainer bdqn
RUN yum -y install yum-utils epel-release
RUN yum -y install nginx
COPY index.html /usr/share/nginx/html
RUN chmod 644 /usr/share/nginx/html/index.html
CMD [“nginx”,“-g”,“daemon off;”]

nginx.repo
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key

通过Git提交到GitLab

在jenkins上构建项目，构建项目时需要注意的地方
1、要在jenkins服务器上安装git yum install -y git
2、要在jenkins服务器上设置免密登录到k8s-master
ssh-keygen
ssh-copy-id 192.168.180.210

cd /data && kubectl create -f nginx-svc.yaml --validate=false