Jenkins + github + docker + k8s + springboot
本文介绍基于k8s、docker、jenkins、springboot构建docker服务。
环境准备

server-1 k8s-master Centos7 ip地址10.12.5.110
server-2 k8s-node Centos7 ip地址10.12.5.115
两台服务执行如下命令

$ setenforce 0
$ systemctl stop firewalld
$ systemctl disable firewalld
server-1 k8s-master 安装

k8s-master节点安装kubernets、docker、etcd、git、maven等软件。
安装docker

$ yum install docker
// 修改docker配置文件 /etc/sysconfig/docker

OPTIONS=‘–registry-mirror=https://docker.mirrors.ustc.edu.cn --selinux-enabled --log-driver=journald --signature-verification=false‘
ADD_REGISTRY=‘–add-registry 10.12.5.110:5000‘
INSECURE_REGISTRY=‘–insecure-registry 10.12.5.110:5000‘

$ systemctl enable docker
$ service docker start
安装registry

$ docker pull registry
$ mkdir -p /data/docker/registry
$ docker run -d -p 5000:5000 -v /data/docker/registry:/var/lib/registry registry
安装etcd

$ yum install etcd
$ vi /etc/etcd/etcd.conf
// 修改如下配置
ETCD_LISTEN_CLIENT_URLS=“http://0.0.0.0:2379”

$ systemctl enable etcd
$ systemctl start etcd
安装kubernetes

$ yum install kubernetes
// 修改配置文件apiserver
$ vi /etc/kubernetes/apiserver
// 修改如下配置
KUBE_API_ADDRESS="–insecure-bind-address=0.0.0.0"
KUBE_API_PORT="–port=8080"
KUBELET_PORT="–kubelet-port=10250"
KUBE_ADMISSION_CONTROL="–admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota"

// 启动服务
$ systemctl enable kube-apiserver kube-controller-manager kube-scheduler
$ systemctl start kube-apiserver kube-controller-manager kube-scheduler
安装git和mvn

$ yum install git
$ yum install maven
server-2 k8s-node 安装

k8s-master节点安装kubernets、docker等软件。
安装docker

$ yum install docker
// 修改docker配置文件 /etc/sysconfig/docker

ADD_REGISTRY=‘–add-registry 10.12.5.110:5000‘
INSECURE_REGISTRY=‘–insecure-registry 10.12.5.110:5000‘

$ systemctl enable docker
$ systemctl start docker
安装kubernetes

$ yum install kubernetes
// 修改配置文件kubelet
$ vi /etc/kubernetes/kubelet
// 修改如下配置
KUBELET_ADDRESS="–address=0.0.0.0"
KUBELET_PORT="–port=10250"
KUBELET_HOSTNAME="–hostname-override=10.12.5.115"
KUBELET_API_SERVER="–api-servers=http://10.12.5.110:8080"
// 修改配置文件config
$ vi /etc/kubernetes/config
// 修改如下配置
KUBE_MASTER="–master=http://10.12.5.110:8080"
// 启动服务
$ systemctl enable kube-proxy kubelet
$ systemctl start kube-proxy kubelet
检查k8s集群运行情况

// 在k8s-master节点执行
$ kubectl get nodes
NAME STATUS AGE
10.12.5.115 Ready 3m
在k8s-master节点builder springcloudenv镜像

• 下载jdk8
  $ mdkir -p /home/docker/docker-jdk8
  $ wget http://javadl.oracle.com/webapps/download/AutoDL?BundleId=233162_512cd62ec5174c3487ac17c61aaa89e8 -O jre-8u161-linux-x64.tar.gz
  $ tar zxvf jre-8u161-linux-x64.tar.gz
• 删除jdk中用不到的文件，尽量减少docker镜像文件大小
  $ cd jre1.8.0_171
  $ rm -rf lib/plugin.jar lib/ext/jfxrt.jar bin/javaws lib/javaws.jar lib/desktop plugin lib/deploy* lib/javafx lib/jfx lib/amd64/libdecora_sse.so lib/amd64/libprism_.so lib/amd64/libfxplugins.so lib/amd64/libglass.so lib/amd64/libgstreamer-lite.so lib/amd64/libjavafx.so lib/amd64/libjfx*.so
• 编写Dockerfile文件
  $ cd /home/docker/docker-jdk8
  $ vi Dockerfile

FROM centos
MAINTAINER by wangtw
RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo ‘Asia/Shanghai‘ >/etc/timezone && yum -y install kde-l10n-Chinese && localedef -c -f UTF-8 -i zh_CN zh_CN.utf8 && mkdir -p /usr/java
COPY jre1.8.0_171 /usr/java/jre1.8.0_171
ENV LANG zh_CN.UTF-8
ENV JAVA_HOME /usr/java/jre1.8.0_171
ENV PATH $JAV{A}_{H}OME/bin:$PATH

• 创建镜像springcloudenv，并上传到registry，确保docker registry已经启动

$ docker build -t springcloudenv .
$ docker tag springcloudenv springcloudenv:v1
$ docker push springcloudenv:v1
// 查看registry中的镜像
$ curl http://10.12.5.110:5000/v2/_catalog
{“repositories”:[“springcloudenv”]}
安装nfs

在k8s-master节点安装nfs server

$ yum install -y nfs-utils rpcbind
$ mkdir -p /data/mysql-pv
$ chmod 777 /data/mysql-pv/
$ mkdir -p /data/nfs
$ chmod 666 /data/nfs/
$ vi /etc/exports
// 增加如下行
/data/nfs 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)
/data/mysql-pv 10.12.0.0/16(rw,no_root_squash,no_all_squash,sync)
$ systemctl enable rpcbind nfs
$ systemctl start rpcbind nfs
$ mkdir -p /data/mysql
$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock
在k8s-node节点安装nfs

$ yum install -y nfs-utils
$ mkdir -p /data/mysql
$ mount -t nfs 10.12.5.110:/data/nfs /data/mysql/ -o proto=tcp -o nolock
安装mysql docker

以持久化方式运行myql
在k8s-master节点创建相关文件

$ mkdir -p /home/k8s/yml/services/mysql
$ cd /home/k8s/yml/services/mysql
$ vi mysql-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql-pv
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 1Gi
persistentVolumeReclaimPolicy: Retain
#storageClassName: nfs
nfs:
path: /data/mysql-pv
server: 10.12.5.110

$ vi mysql-pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mysql-pvc
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
#storageClassName: nfs

$ vi mysql.yaml

apiVersion: v1
kind: Service
metadata:
name: mysql
spec:
ports:

• port: 3306
  selector:
  app: mysql

---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: mysql
spec:
replicas: 1
selector:
matchLabels:
app: mysql
template:
metadata:
labels:
app: mysql
spec:
containers:
- name: mysql
image: mysql:5.6
env:
- name: MYSQL_ROOT_PASSWORD
value: password
ports:
- containerPort: 3306
name: mysql
volumeMounts:
- name: mysql-persistent-storage
mountPath: /var/lib/mysql
volumes:
- name: mysql-persistent-storage
persistentVolumeClaim:
claimName: mysql-pvc
创建mysql service和pod

$ cd /home/k8s/yml/services/mysql
$ kubectl create -f mysq-pv.yaml
$ kubectl get pv

NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE
mysql-pv 1Gi RWO Retain Bound default/mysql-pvc 2d

$ kubectl create -f mysq-pvc.yaml
$ kubectl get pvc

NAME STATUS VOLUME CAPACITY ACCESSMODES AGE
mysql-pvc Bound mysql-pv 1Gi RWO 2d

$ kubectl create -f mysq.yaml
$ kubectl get service

NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mysql 10.254.151.113 3306/TCP 2d

$ kubectl get pod

NAME READY STATUS RESTARTS AGE
mysql-3827607452-hd5ct 1/1 Running 80 19h
k8s-master节点安装和配置jenkins

$ mkdir /home/jenkins
$ cd
$ vi .bash_profile
export JENKINS_HOME=/home/jenkins
$ cd /home/jenkins
$ wget http://mirrors.jenkins.io/war-stable/latest/jenkins.war
$ java -jar jenkins.war --httpPort=9090&
浏览器打开地址：http://10.12.5.110:9090
把文件/home/jenkins/secrets/initialAdminPassword的内容复制到安装页面
解决插件安装离线问题

打开地址：http://10.12.5.110:9090/pluginManager/advanced
把https://updates.jenkins.io/update-center.json修改为http://updates.jenkins.io/update-center.json

安装插件：Publish Over SSH
创建jenkins任务,源代码在github上

• 任务名称：springboottest
• 任务类型：自由风格
• 创建github访问Credential

Credential类型：username with password

• 源码管理 选择git
  Repository URL 值为https://github.com/wangtiewu/springboottest.git
  Credentials 值为上一步创建的 Credential
• 创建构建脚本

mvn clean package
tag=$( date +‘%Y%m%d%H%M‘ )
app=springcloudtest
echo KaTeX parse error: Expected group after '_' at position 14: tag > version_̲app.txt
docker_name=$app
docker build -t $docke{r}_{n}ame:$tag .
docker push $docke{r}_{n}ame:$tag

• 创建ssh server

系统管理-〉系统设置菜单-〉publish over ssh

• 增加构建后操作，Send build artifacts over SSH

SSH Server:name 值为k8s master
Transfer Set:Source files 值为version_springcloudtest.txt
Transfer Set:Remote directory 值为springcloudtest
Transfer Set:Exec command 值为
cd /home/k8s/yml/services/springcloudtest
app=springcloudtest
deploy_file=KaTeX parse error: Expected group after '_' at position 30: …on_file=version_̲app.txt
patch_file=$app.patchne{w}_{v}ersion=$( cat $versio{n}_{f}ile)ol{d}_{v}ersion=$( cat $deploy_file | grep image | awk -F “:” ‘{print KaTeX parse error: Expected 'EOF', got '}' at position 2: 3}̲‘ ) sed -i "s/old_version/$new_version/" $deplo{y}_{f}iledeploymen{t}_{n}um=$( /usr/bin/kubectl get deployment | grep springcloudtest | wc -l )
if [ $deploymen{t}_{n}um-eq1];then{p}_{o}l{d}_{v}ersion=$( cat KaTeX parse error: Can't use function '\"' in math mode at position 79: … p_new_version=\̲"̲app:KaTeX parse error: Can't use function '\"' in math mode at position 12: new_version\̲"̲ sed -i "s/p_old_version/$p_new_version/" $patc{h}_{f}ilepatch=$( cat $patch_file )
/usr/bin/kubectl patch deployment $app -p $patch
else
/usr/bin/kubectl create -f $deploy_file
fi
创建springclouttest service和pod

$ mkdir -p /home/k8s/yml/services/springcloudtest
$ cd /home/k8s/yml/services/springcloudtest
$ vi springcloudtest-svr.yaml

apiVersion: v1
kind: Service
metadata:
name: springcloudtest
spec:
ports:

• name: springcloudtest-svr
  port: 9091
  nodePort: 30000
  targetPort: 9091
  selector:
  app: springcloudtest
  type: NodePort

$ vi springcloudtest.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: springcloudtest
spec:
replicas: 1
template:
metadata:
labels:
app: springcloudtest
spec:
containers:
- name: springclouttest
image: springcloudtest:201805272108
ports:
- containerPort: 9091
protocol: TCP

$ vi springcloudtest.patch

{“spec”:{“template”:{“spec”:{“containers”:[{“name”:“springclouttest”,“image”:“springcloudtest:201805052039”}]}}}}
构建jenkins任务：springclouttest

构建完成后，kubectl get pod 查看

springcloudtest-1053936621-w19sh 1/1 Running 1 1d
创建springclouttest service

$ cd /home/k8s/yml/services/springcloudtest
$ kubectl create -f springcloudtest-svr.yaml
查看服务状态
$ kubectl get service
springcloudtest 10.254.126.13 9091:30000/TCP 45m
在k8s-node查看容器情况
$ docker ps

CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
269366fa1b49 mysql:5.6 “docker-entrypoint…” About an hour ago Up About an hour k8s_mysql.affc4af4_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_404d6f6a
2ff21fb61984 springcloudtest:201805272108 “/bin/sh -c ‘java …” About an hour ago Up About an hour k8s_springclouttest.c0ee0b5d_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_a99e1b37
88e9ba47faf1 registry.access.redhat.com/rhel7/pod-infrastructure:latest “/usr/bin/pod” About an hour ago Up About an hour k8s_POD.389b0ddb_mysql-3827607452-lp1cp_default_15ace557-60ed-11e8-aa6f-0800276c68b7_7387fdc6
130a60acd94d registry.access.redhat.com/rhel7/pod-infrastructure:latest “/usr/bin/pod” About an hour ago Up About an hour k8s_POD.2aaa0bac_springcloudtest-1053936621-w19sh_default_a320f883-61b2-11e8-aa6f-0800276c68b7_fa8aba88

解决 NodePort方式下,Node所在节点可以访问服务,节点外客户端请求服务都会失败问题
$ iptables -P FORWARD ACCEPT
创建数据库和表

$ docker exec -it 269366fa1b49 /bin/bash
$ mysql -uroot -p
输入密码

mysql>create database test1 default charset utf8 COLLATE utf8_general_ci;
mysql>use test1
mysql>create table HELLO(ID bigint, NAME varchar(128), primary key (ID));
测试服务

浏览器打开 http://10.12.5.115:30000/
浏览器显示 Hello World!

$ curl --data ‘{“id”:1,“name”:“Hello World”}‘ -H “Content-type: Application/json” http://10.12.5.115:30000/create/hello