实战项目部署应用到kubernetes流程(jenkins+docker+k8s)

说明通过jenkins构建java应用程序发布到k8s集群中本文已一个大数据的java项目来演示构建部署过程支持发布和回滚支持一套模板应用不同项目k8s基础准备创建项目名称空间[root@k8s-master ~]# kubectl create ns architectnamespace/architect created[root@k8s-m...

weixin_30591551

1222人浏览 · 2019-07-24 11:09:00

weixin_30591551 · 2019-07-24 11:09:00 发布

说明

通过jenkins构建java应用程序发布到k8s集群中

本文已一个大数据的java项目来演示构建部署过程

支持发布和回滚

支持一套模板应用不同项目

k8s基础准备

创建项目名称空间

[root@k8s-master ~]# kubectl create ns architect
namespace/architect created

[root@k8s-master ~]# kubectl label namespace architect istio-injection=enabled
namespace/architect labeled
[root@k8s-master ~]# kubectl get namespace -L istio-injection
NAME           STATUS   AGE     ISTIO-INJECTION
architect      Active   87s     enabled
default        Active   39d     enabled
istio-system   Active   7d17h   
kube-public    Active   39d     
kube-system    Active   39d     
[root@k8s-master ~]#

创建全局配置文件

＃创建大数据配置文件
[root@k8s-master ~]# ls bigdata-counter
core-site.xml  dmp_operator1.keytab  hbase-site.xml  hdfs-site.xml  krb5.conf
[root@k8s-master ~]#
[root@k8s-master ~]# kubectl create configmap bigdata-config --from-file=/root/bigdata-counter/ -n architect 
configmap/bigdata-config created
[root@k8s-master ~]# 
#全局环境变量
[root@k8s-master ~]# cat yaml/global-config.yaml 
apiVersion: v1
kind: ConfigMap
metadata:
  name: global-config
  namespace: architect
data:
  CONFIG_SERVER_URL: http://config.ms.xx.com
  CONFIG_SERVER_USER: myconfig
  ILOG_BOOTSTRAP_SERVERS: bootstrap.servers=192.168.1.122:9092
  CONFIG_SERVER_PWDS: mc123456

[root@k8s-master ~]# 
[root@k8s-master ~]# kubectl apply -f yaml/global-config.yaml 
configmap/global-config created
[root@k8s-master ~]# kubectl get configmap -n architect
NAME DATA AGE
bigdata-config 4 2m33s
global-config 4 3s
[root@k8s-master ~]#

创建habror认证信息

[root@k8s-master ~]# kubectl create secret docker-registry registry-secret --docker-server=dev-hub.xx.net --docker-username=admin --docker-password=Harbor12345 --docker-email=admin@dev-hub.xx.net -n architect
secret/registry-secret created
[root@k8s-master ~]# kubectl get secret -n architect
NAME                  TYPE                                  DATA   AGE
default-token-5wsxl   kubernetes.io/service-account-token   3      2m24s
istio.default         istio.io/key-and-cert                 3      2m24s
registry-secret       kubernetes.io/dockerconfigjson        1      10s

项目deployment文件

可以做成模本，通过变量替换形式．后面有讲到

[root@k8s-master ~]# cat bigdata-interface-deploy.yaml 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: bigdata-interface
    version: default
  name: bigdata-interface
  namespace: architect
spec:
  replicas: 1
  selector:
    matchLabels:
      app: bigdata-interface
      version: default
  template:
    metadata:
      labels:
        app: bigdata-interface
        version: default
    spec:
      containers:
      - env:
        - name: izone
        - name: MY_SERVICE_NAME
          value: bigdata-interface
        - name: ilogEnvs
          value: izone,buildNumber
        - name: ILOG_BOOTSTRAP_SERVERS
          valueFrom:
            configMapKeyRef:
              key: ILOG_BOOTSTRAP_SERVERS
              name: global-config
        - name: CONFIG_SERVER_URL
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_URL
              name: global-config
        - name: CONFIG_SERVER_USER
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_USER
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: JAR_BOOT_ARG
          value: --spring.cloud.config.fail-fast=true
        - name: JDK_HEAP_OPTS
          value: -Xms256m -Xmx256m
        image: dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b63
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 2
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        name: bigdata-interface
        ports:
        - containerPort: 80
          name: http
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        volumeMounts:
        - mountPath: /opt/jdk
          name: jdk-path
        - mountPath: /srv/applogs
          name: applogs
      imagePullSecrets:
      - name: registry-secret
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes:
      - hostPath:
          path: /usr/local/jdk/default
          type: ""
        name: jdk-path
      - hostPath:
          path: /data/applogs
          type: ""
        name: applogs
[root@k8s-master ~]#

jenkins pipeline 模板文件

node {
    
   
     //项目名称
    String APPNAME='bigdata-interface'
     //名称空间
    String NAMESPACE='architect'
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
    


      properties([
         parameters([
         string(name:'GIT_BRANCH_TAG', defaultValue: 'master',description:'', trim: false) 
    ])])
           

   
    stage('git clone') {
        checkout([$class: 'GitSCM', branches: [[name: params.GIT_BRANCH_TAG]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git']]])
    }
       
    stage('maven build') {
        
         sh 'mvn clean package -U'  
         
    }
      
    stage('docker build && push habror'){
             sh "cd ${WORKSPACE}"
             restapi = '`ls -d *-restapi||echo .`'
           
             sh "ls -l ${restapi}/target/"
             sh "echo ${restapi}"
             sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1"
             sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target"
             sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
             sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}"
             sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
         
        }
    stage('deploy'){
             echo "${params.GIT_BRANCH_TAG}"
             echo "${BUILD_NUMBER}"
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    } 
  
 }

　进一步改造pipeline

node {
      
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
     
 
 
      properties([
         parameters([
			string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
			string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
			string(defaultValue: 'git@git.jiatuiyun.net:bigdata-center/bigdata-interface.git', description: 'git仓库地址', name: 'XBUILD_REPO', trim: false),
			string(defaultValue: 'master', description: '版本名称', name: 'XBUILD_BRANCHE', trim: false)
    ])])
            
 
    
    stage('git clone') {
        checkout([$class: 'GitSCM', branches: [[name: params.XBUILD_BRANCHE]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'b6a484b7-cbab-4215-9e93-2271d7a333bc', url: params.XBUILD_REPO]]])
    }
        
    stage('maven build') {
         
         sh 'mvn clean package -U' 
          
    }
       
    stage('docker build && push habror'){
             sh "cd ${WORKSPACE}"
             restapi = '`ls -d *-restapi||echo .`'
            
             sh "ls -l ${restapi}/target/"
             sh "echo ${restapi}"
             sh "curl -o ${restapi}/target/Dockerfile http://192.168.1.55:8089/files/Dockerfile-java.v1"
			 NAMESPACE = params.XBUILD_TEAM
			 APPNAME = params.XBUILD_DEPLOY
			
             sh "docker build -t ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${restapi}/target"
             sh "docker tag ${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
             sh "docker login ${HarborUrl} -u ${HarborUser} -p ${HarborPasswd}"
             sh "docker push ${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER}"
          
        }
    stage('deploy'){
			 NAMESPACE = params.XBUILD_TEAM
			 APPNAME = params.XBUILD_DEPLOY
             echo "${params.GIT_BRANCH_TAG}"
             echo "${BUILD_NUMBER}"
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    } 
   
 }

jenkins上创建自由风格项目

效果如下支持参数化构建。但每项目单独JOB

查看镜像仓库已经推送正常

项目回滚

node {
    
      properties([
         parameters([
        string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
		string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
		string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)	 

    ])])   

    stage('deploy'){
             
             echo "${BUILD_NUMBER}"
			  NAMESPACE = params.XBUILD_TEAM
			 APPNAME = params.XBUILD_DEPLOY
			 BUILD_NUMBER = params.XBUILD_NUMBER
			 echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} "
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             //updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             //test
    }
  
 }

jenkins效果如下：输入namespace和deployment 名称和上次构建版本

查看k8s集群

可以灵活的在已知版本回滚操作

[root@k8s-master ~]# kubectl get deploy -n architect -o wide
NAME                READY   UP-TO-DATE   AVAILABLE   AGE     CONTAINERS          IMAGES                                                           SELECTOR
bigdata-interface   0/1     1            0           3h12m   bigdata-interface   dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b1   app=bigdata-interface,version=default
[root@k8s-master ~]# kubectl get pods -n architect
NAME                                 READY   STATUS    RESTARTS   AGE
bigdata-interface-6bf7b5646d-sf9j5   1/2     Running   0          24s
[root@k8s-master ~]#

通过以上实践，可以将deployment做成模板

deployment模板文件

-bash-4.2$ cat k8s-deployment.tpl 
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: {APPNAME}
    version: default
  name: {APPNAME}
  namespace: {NAMESPACE}
spec:
  replicas: 1
  selector:
    matchLabels:
      app: {APPNAME}
      version: default
  template:
    metadata:
      labels:
        app: {APPNAME}
        version: default
    spec:
      containers:
      - env:
        - name: izone
        - name: MY_SERVICE_NAME
          value: {APPNAME}
        - name: ilogEnvs
          value: izone,buildNumber
        - name: ILOG_BOOTSTRAP_SERVERS
          valueFrom:
            configMapKeyRef:
              key: ILOG_BOOTSTRAP_SERVERS
              name: global-config
        - name: CONFIG_SERVER_URL
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_URL
              name: global-config
        - name: CONFIG_SERVER_USER
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_USER
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: CONFIG_SERVER_PWDS
          valueFrom:
            configMapKeyRef:
              key: CONFIG_SERVER_PWDS
              name: global-config
        - name: JAR_BOOT_ARG
          value: --spring.cloud.config.fail-fast=true
        - name: JDK_HEAP_OPTS
          value: -Xms256m -Xmx256m
        image: {HarborUrl}/{NAMESPACE}/{APPNAME}:{NAMESPACE}_b{BUILD_NUMBER}
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 2
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        name: {APPNAME}
        ports:
        - containerPort: 80
          name: http
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /actuator/info
            port: 80
            scheme: HTTP
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 30
        volumeMounts:
        - mountPath: /opt/jdk
          name: jdk-path
        - mountPath: /srv/applogs
          name: applogs
      imagePullSecrets:
      - name: registry-secret
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      volumes:
      - hostPath:
          path: /usr/local/jdk/default
          type: ""
        name: jdk-path
      - hostPath:
          path: /data/applogs
          type: ""
        name: applogs
-bash-4.2$

通过模板动态生成项目的deploy文件．进行升级回滚．

比如下面的：

node {
    String HarborUrl='dev-hub.jiatuiyun.net'
    String HarborUser='admin'
    String HarborPasswd='Harbor12345'
    String KubernetHost = 'root@192.168.100.101'
      properties([
         parameters([
        string(defaultValue: 'architect', description: 'git组', name: 'XBUILD_TEAM', trim: false),
		string(defaultValue: 'bigdata-interface', description: '项目名', name: 'XBUILD_DEPLOY', trim: false),
		string(defaultValue: '1', description: '镜像标签', name: 'XBUILD_NUMBER', trim: false)	 

    ])])   

    stage('deploy'){
             
             echo "${BUILD_NUMBER}"
			  NAMESPACE = params.XBUILD_TEAM
			 APPNAME = params.XBUILD_DEPLOY
			 BUILD_NUMBER = params.XBUILD_NUMBER
			 echo "----${params.XBUILD_NUMBER}---${params.XBUILD_DEPLOY} ----${params.XBUILD_TEAM} "
             //kubectl set image deployment/bigdata-interface bigdata-interface=dev-hub.jiatuiyun.net/architect/bigdata-interface:architect_b48 -n architect
             updateImage="kubectl set image deployment/${APPNAME} --namespace=${NAMESPACE}  ${APPNAME}=${HarborUrl}/${NAMESPACE}/${APPNAME}:${NAMESPACE}_b${BUILD_NUMBER} --record"
             command="sed -e 's#{APPNAME}#${APPNAME}#g;s#{NAMESPACE}#${NAMESPACE}#g;s#{HarborUrl}#${HarborUrl}#g;s#{BUILD_NUMBER}#${BUILD_NUMBER}#g' /data/jenkins_slave/k8s-deployment.tpl > /data/jenkins_slave/${APPNAME}-deploy.yaml"
             sh "${command}"
             sh "scp -P 22 /data/jenkins_slave/${APPNAME}-deploy.yaml ${KubernetHost}:/root/ "
             //sh "ssh -p 22 ${KubernetHost} ${updateImage}"
             sh "ssh -p 22 ${KubernetHost} kubectl apply -f /root/${APPNAME}-deploy.yaml"
             //test
    }
  
 }

转载于:https://www.cnblogs.com/xuliang666/p/11236754.html