k8s1.16.0-k8s的jenkins部署java项目到k8s集群cicd(ci成,cd手动部署的)

注意: 本文档只是实现了ci,cd是通过ci生成的镜像,再手工再k8s-master执行的部署(只因pod部署的jenkins连接k8s的认证不知怎么操作,若jenkins是单独部署在k8s-master机器上,能直接在master执行kubectl命令就没这个问题了)

1.规划: 

192.168.171.128    master   mysql  nfs

192.168.171.129    node1

192.168.171.130    node2

192.168.171.131    githabor docker-habor  

2.在192.168.171.131上: 安装githarbor  docker-harbor

1)在192.168.171.131上安装docker-ce19.03

[root@localhost ~]# ls

docker19.03-ce_lixian.tar.gz

[root@localhost ~]# tar -zxf docker19.03-ce_lixian.tar.gz

[root@localhost ~]# ls

docker19.03-ce_lixian  docker19.03-ce_lixian.tar.gz

[root@localhost ~]# cd docker19.03-ce_lixian

[root@localhost docker19.03-ce_lixian]# ls

containerd.io-1.2.6-3.3.el7.x86_64.rpm  docker-ce-19.03.2-3.el7.x86_64.rpm  docker-ce-cli-19.03.2-3.el7.x86_64.rpm

[root@localhost docker19.03-ce_lixian]# yum -y localinstall *.rpm

[root@localhost docker19.03-ce_lixian]# systemctl start docker

[root@localhost docker19.03-ce_lixian]# systemctl enable docker

[root@localhost docker19.03-ce_lixian]# docker info

Client:

 Debug Mode: false

Server:

 Containers: 0

  Running: 0

  Paused: 0

  Stopped: 0

 Images: 0

 Server Version: 19.03.2

[root@localhost docker19.03-ce_lixian]# cd

2)安装docker-compose,harbor依赖于compose

[root@localhost ~]# rz

[root@localhost ~]# ls

docker19.03-ce_lixian  docker19.03-ce_lixian.tar.gz  docker-compose-bin.tar.gz

[root@localhost ~]# chmod +x docker-compose

[root@localhost ~]# mv docker-compose /usr/bin/

[root@localhost ~]# docker-compose -v

docker-compose version 1.21.0, build 5920eb0

3)安装docker-harbor1.7.5  推荐2核4G

[root@localhost ~]# rz

[root@localhost ~]# ls

docker19.03-ce_lixian  docker19.03-ce_lixian.tar.gz  docker-compose-bin.tar.gz  harbor-offline-installer-v1.7.5.tgz

[root@localhost ~]# tar -zxf harbor-offline-installer-v1.7.5.tgz

[root@localhost ~]# ls

docker19.03-ce_lixian  docker19.03-ce_lixian.tar.gz  docker-compose-bin.tar.gz  harbor  harbor-offline-installer-v1.7.5.tgz

[root@localhost ~]# cd harbor

[root@localhost harbor]# ls

common                          docker-compose.clair.yml   docker-compose.yml  harbor.v1.7.5.tar.gz  LICENSE              prepare

docker-compose.chartmuseum.yml  docker-compose.notary.yml  harbor.cfg          install.sh            open_source_license

[root@localhost harbor]# vim harbor.cfg

hostname = 192.168.171.131    #设置为使用ip访问

harbor_admin_password = Harbor12345   #默认登录密码

[root@localhost harbor]# ./prepare     #准备安装环境脚本

[root@localhost harbor]# ./install.sh    #安装harbor       安装完后,会自动启动下面容器:

[root@localhost harbor]# docker-compose ps

       Name                     Command                       State                                        Ports                              

----------------------------------------------------------------------------------------------------------------------------------------------

harbor-adminserver   /harbor/start.sh                 Up (healthy)                                                                            

harbor-core          /harbor/start.sh                 Up (healthy)                                                                            

harbor-db            /entrypoint.sh postgres          Up (healthy)            5432/tcp                                                        

harbor-jobservice    /harbor/start.sh                 Up                                                                                      

harbor-log           /bin/sh -c /usr/local/bin/ ...   Up (health: starting)   127.0.0.1:1514->10514/tcp                                       

harbor-portal        nginx -g daemon off;             Up (healthy)            80/tcp                                                          

nginx                nginx -g daemon off;             Up (healthy)            0.0.0.0:443->443/tcp, 0.0.0.0:4443->4443/tcp, 0.0.0.0:80->80/tcp

redis                docker-entrypoint.sh redis ...   Up                      6379/tcp                                                        

registry             /entrypoint.sh /etc/regist ...   Up (healthy)            5000/tcp                                                        

registryctl          /harbor/start.sh                 Up (healthy)                                                    

[root@localhost harbor]# cd

浏览器访问docker-harbor  http://192.168.171.131/ 

 输入默认用户名和密码: admin/Harbor12345,登录后,如下:

创建一个公用的仓库即可,因为下面pipeline代码中会推送到该仓库中。

 

4)安装git代码版本仓库(和gitlab差不多,比gitlab简单轻量)

[root@localhost ~]# yum -y install git

设置git代码仓库管理用户和密码:git/123

[root@localhost ~]# useradd git

[root@localhost ~]# echo "123" |passwd --stdin git

[root@localhost ~]# su - git

[git@localhost ~]$ mkdir demo.git    #创建一个空的仓库

[git@localhost ~]$ ls

demo.git

[git@localhost ~]$ cd demo.git/

[git@localhost demo.git]$ ls

[git@localhost demo.git]$ git --bare init    #初始化仓库

[git@localhost demo.git]$ ls

branches  config  description  HEAD  hooks  info  objects  refs

5)随便找个客户端测试git版本管理仓库,以master作为git客户端测试:

[root@master ~]# yum -y install git   #安装git客户端

[root@master ~]# git clone git@192.168.171.131:/home/git/demo.git

git@192.168.171.131's password: 123

warning: You appear to have cloned an empty repository.   #提示为空,因为仓库里没有东西

[root@master ~]# ls demo/

[root@master ~]# cd demo/

[root@master demo]# ls

[root@master demo]# echo 123 >a.txt

[root@master demo]# cat a.txt

123

[root@master demo]# git config --global user.email "you@example.com"

[root@master demo]# git config --global user.name "Your Name"

[root@master demo]# git add .

[root@master demo]# git commit -m "test"

[root@master demo]# git remote -v    #查看远程仓库别名设置

origin git@192.168.171.131:/home/git/demo.git (fetch)

origin git@192.168.171.131:/home/git/demo.git (push)

[root@master demo]# git push origin master

git@192.168.171.131's password:  123

Counting objects: 3, done.

Writing objects: 100% (3/3), 205 bytes | 0 bytes/s, done.

Total 3 (delta 0), reused 0 (delta 0)

To git@192.168.171.131:/home/git/demo.git

 * [new branch]      master -> master

验证推送仓库成功,将现有的仓库删除,重新clone远程仓库,有新代码即可,如下:

[root@master demo]# cd ..

[root@master ~]# rm -rf demo/

[root@master ~]# git clone git@192.168.171.131:/home/git/demo.git

Cloning into 'demo'...

git@192.168.171.131's password: 123

[root@master ~]# cd demo/

[root@master demo]# ls

a.txt

[root@master demo]# cat a.txt

123

[root@master demo]# cd ..

客户端免交互拉取代码配置:在客户端上生成秘钥对,将公钥发给git服务端即可。

[root@master ~]# ssh-keygen   一路回车

[root@master ~]# ls /root/.ssh/

id_rsa  id_rsa.pub  known_hosts

[root@master ~]# ssh-copy-id git@192.168.171.131

git@192.168.171.131's password:123

[root@master ~]# rm -rf demo/

[root@master ~]# git clone git@192.168.171.131:/home/git/demo.git   #可以免交互拉取git仓库代码

[root@master ~]# ls demo/

a.txt

[root@master ~]# cat demo/a.txt

123

2.在k8s集群中部署jenkins

[root@k8s-master ~]# kubectl get nodes

NAME        STATUS   ROLES    AGE   VERSION

k8s-node1   Ready    <none>   57d   v1.16.0

k8s-node2   Ready    <none>   57d   v1.16.0

jenkins数据需要持久化且共享存储,因为不知道会把jenkins分配到哪个节点,或者运行jenkins的节点挂了漂移到另一个节点时,都需要共享数据,所以,需要搭建nfs

1)在192.168.171.128-master上安装nfs:

[root@k8s-master ~]# yum -y install nfs-utils #客户端和服务端都需要安装(k8s集群机器也安装)

[root@k8s-master ~]# vim /etc/exports

/home/jenkins *(no_root_squash,rw)

/home/maven *(no_root_squash,rw)

[root@k8s-master ~]# mkdir /home/jenkins -p

[root@k8s-master ~]# mkdir /home/maven -p

[root@k8s-master ~]# exportfs -rv

[root@k8s-master ~]# systemctl restart nfs

[root@k8s-master ~]# systemctl enable nfs

2)K8s中部署jenkins: (以deployment方式部署,共享存储nfs和pv)

[root@k8s-master ~]# ls jenkins_yaml_k8s1.16.tar.gz

jenkins_yaml_k8s1.16.tar.gz

[root@k8s-master ~]# tar -zxf jenkins_yaml_k8s1.16.tar.gz

[root@k8s-master ~]# cd jenkins_yaml

[root@k8s-master jenkins_yaml]# ls

default-sa.yaml  deploy.yaml  pv.yaml  sa.yaml

[root@k8s-master jenkins_yaml]# cat default-sa.yaml

---

kind: Role

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

  name: default-role

rules:

- apiGroups: [""]

  resources: ["pods"]

  verbs: ["create","delete","get","list","patch","update","watch"]

- apiGroups: [""]

  resources: ["pods/exec"]

  verbs: ["create","delete","get","list","patch","update","watch"]

- apiGroups: [""]

  resources: ["pods/log"]

  verbs: ["get","list","watch"]

- apiGroups: [""]

  resources: ["secrets"]

  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: RoleBinding

metadata:

  name: default-rolebinding

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: Role

  name: default-role

subjects:

- kind: ServiceAccount

  name: default

  namespace: default

[root@k8s-master jenkins_yaml]# cat sa.yaml

---

apiVersion: v1

kind: ServiceAccount

metadata:

  name: jenkins-master

  namespace: default

---

kind: Role

apiVersion: rbac.authorization.k8s.io/v1beta1

metadata:

  name: jenkins-master

rules:

- apiGroups: [""]

  resources: ["pods"]

  verbs: ["create","delete","get","list","patch","update","watch"]

- apiGroups: [""]

  resources: ["pods/exec"]

  verbs: ["create","delete","get","list","patch","update","watch"]

- apiGroups: [""]

  resources: ["pods/log"]

  verbs: ["get","list","watch"]

- apiGroups: [""]

  resources: ["secrets"]

  verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1beta1

kind: RoleBinding

metadata:

  name: jenkins-master

roleRef:

  apiGroup: rbac.authorization.k8s.io

  kind: Role

  name: jenkins-master

subjects:

- kind: ServiceAccount

  name: jenkins-master

  namespace: default

[root@k8s-master jenkins_yaml]# cat pv.yaml

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: jenkins-master-vol

spec:

  capacity:

    storage: 5Gi

  accessModes:

  - ReadWriteMany

  nfs:

    path: /home/jenkins

    server: 192.168.171.128

  persistentVolumeReclaimPolicy: Recycle

---

apiVersion: v1

kind: PersistentVolume

metadata:

  name: maven-repository

spec:

  capacity:

    storage: 5Gi

  accessModes:

  - ReadWriteMany

  nfs:

    path: /home/maven

    server: 192.168.171.128

  persistentVolumeReclaimPolicy: Recycle

---

kind: PersistentVolumeClaim

apiVersion: v1

metadata:

  name: jenkins-master-claim

spec:

  accessModes:

    - ReadWriteMany

  resources:

    requests:

      storage: 5Gi

---

kind: PersistentVolumeClaim

apiVersion: v1

metadata:

  name: maven-repository-claim

spec:

  accessModes:

    - ReadWriteMany

  resources:

    requests:

      storage: 5Gi

[root@k8s-master jenkins_yaml]# cat deploy.yaml

---

apiVersion: apps/v1

kind: Deployment

metadata:

  name: jenkins-master

spec:

  replicas: 1

  strategy:

    type: Recreate

  selector:

    matchLabels:

      name: jenkins-master

  template:

    metadata:

      labels:

        name: jenkins-master

    spec:

      securityContext:

        fsGroup: 1000

      containers:

        - name: jenkins-master

          image: k8s-jenkins-master:2.164.3

          imagePullPolicy: IfNotPresent

          ports:

            - containerPort: 8080

              name: http

            - containerPort: 50000

              name: agent

          volumeMounts:

            - name: jenkins-master-vol

              mountPath: /var/jenkins_home

            - name: maven-repository

              mountPath: /opt/maven/repository

            - name: docker

              mountPath: /usr/bin/docker

            - name: docker-sock

              mountPath: /var/run/docker.sock

      volumes:

        - name: jenkins-master-vol

          persistentVolumeClaim:

            claimName: jenkins-master-claim

        - name: maven-repository

          persistentVolumeClaim:

            claimName: maven-repository-claim

        - name: docker

          hostPath:

            path: /usr/bin/docker

        - name: docker-sock

          hostPath:

            path: /var/run/docker.sock

      serviceAccount: "jenkins-master"

      imagePullSecrets:

        - name: harborsecret

---

apiVersion: v1

kind: Service

metadata:

  name: jenkins-master

spec:

  type: NodePort

  ports:

    - port: 8080

      name: http

      targetPort: 8080

      nodePort: 30006

    - port: 50000

      name: agent

      #nodePort: 30007

      targetPort: 50000

  selector:

    name: jenkins-master

[root@k8s-master jenkins_yaml]# ls /opt/k8s-jenkins-master_2.164.3.tar  #上传基础jenkins的镜像

[root@k8s-master jenkins_yaml]# docker load -i /opt/k8s-jenkins-master_2.164.3.tar   #其他各个节点也都load一下

[root@k8s-master jenkins_yaml]# docker images |grep jenkins

k8s-jenkins-master                          2.164.3             e0da318a8e6e        2 minites ago        1.16GB

[root@k8s-master jenkins_yaml]# kubectl apply -f .

[root@k8s-master jenkins_yaml]# kubectl get pod,svc -o wide

NAME                        READY   STATUS    RESTARTS   AGE   IP            NODE        NOMINATED NODE   READINESS GATES

pod/jenkins-master-78577cdf6c-clcdd   1/1     Running   0          28m   10.244.1.46   k8s-node2   <none>           <none>

NAME                     TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                          AGE   SELECTOR

service/jenkins-master   NodePort    10.0.0.64    <none>        8080:30006/TCP,50000:32224/TCP   15h   name=jenkins-master

service/kubernetes       ClusterIP   10.0.0.1     <none>        443/TCP                          60d   <none>

[root@k8s-master jenkins_yaml]# kubectl get pv,pvc

NAME                  CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                  STORAGECLASS   REASON   AGE

persistentvolume/jenkins-master-vol   5Gi      RWX       Recycle      Bound    default/maven-repository-claim                           20h

persistentvolume/maven-repository   5Gi     RWX         Recycle       Bound    default/jenkins-master-claim                             20h

NAME                            STATUS   VOLUME               CAPACITY   ACCESS MODES   STORAGECLASS   AGE

persistentvolumeclaim/jenkins-master-claim     Bound    maven-repository     5Gi        RWX                           20h

persistentvolumeclaim/maven-repository-claim   Bound    jenkins-master-vol   5Gi        RWX                           20h

[root@master ~]# kubectl logs -f jenkins-master-78577cdf6c-clcdd

......

Jenkins initial setup is required. An admin user has been created and a password generated.

Please use the following password to proceed to installation:

b04b5b8345344783a9a32237aeeb0373

This may also be found at: /var/jenkins_home/secrets/initialAdminPassword

[root@k8s-master jenkins_yaml]# kubectl exec -it jenkins-master-78577cdf6c-clcdd bash

root@jenkins-master-78577cdf6c-clcdd:/# ping www.baidu.com

PING www.a.shifen.com (61.135.169.125) 56(84) bytes of data.

64 bytes from 61.135.169.125 (61.135.169.125): icmp_seq=1 ttl=127 time=49.5 ms

root@jenkins-master-78577cdf6c-clcdd:/# ping kubernetes

PING kubernetes.default.svc.cluster.local (10.0.0.1) 56(84) bytes of data.

64 bytes from kubernetes.default.svc.cluster.local (10.0.0.1): icmp_seq=1 ttl=64 time=0.175 ms

64 bytes from kubernetes.default.svc.cluster.local (10.0.0.1): icmp_seq=2 ttl=64 time=0.061 ms

浏览器访问jenkins: http://192.168.171.130:30006   

下面路径是之前的jenkins截图,

路径实际应该在: /var/jenkins_home/secrets/initialAdminPassword

 

点击选择插件来安装,如下: 

 先不安装任何插件,后面再安装,如下:

创建第一个管理员用户,admin/123456

 

将插件管理处的高级处https改为http(为了提高安装插件速度)——点击check now,有时候可能需要等一段时间,再选择可用插件,安装相关使用插件即可,如下: 

 将https改为http——点击check now,有时候可能需要等一段时间,再选择可用插件,安装相关使用插件即可。

 3.安装jenkins中的相关插件

1)下面安装git插件: (最好先上传git相关难安装的插件,再用平台页面安装git插件)如下:

安装git相关难安装的插件,如下:

选择好插件后,点击upload上传后会自动安装该上传插件,如下: 

 

 上面报错是因为安装git插件时候因为依赖关系,有先后顺序,这个不影响,哪个没有安装,再重新下载相关插件,重新安装即可,直到全部安装完成即可。

同理,将git相关依赖的难安装的插件,先手工离线安装,可以看到git插件已经安装完成,如果没有安装完成,然后再用该平台安装git插件:在available出选择Git plugin勾选并安装即可。

2)安装kubernetes插件:

也是先手动上传安装难安装的依赖插件,如下:

 所有相关依赖难安插件安装完成后,如下:

 上面报错是因为安装git插件时候因为依赖关系,有先后顺序,这个不影响,哪个没有安装,再重新下载相关插件,重新安装即可,直到全部安装完成即可。

在installed中可以看到已经安装好kubernetes 插件:

 

如果没有安装好,就在available中选择Kubernetes plugin再进行安装一下即可。

3)安装pipeline插件:   也是先手动上传安装难安装的依赖插件,如下:

所有相关依赖难安插件安装完成后,如下:因为可能依赖关系顺序不同,下面有的有报错,后面安装后又将报错的重新安装了一下,成功。

 

 安装完成后,在install中可以看到pipeline的相关安装,但是Pipeline还没有安装,需要在平台上available中勾选Pipeline再进行安装,如下:

安装完成后,在installed中可以看到Pipeline,如下:

4).安装jenkins将资源部署在k8s平台的插件:  Kubernetes_Continuos_Deploy插件:

 如果两个依赖插件都安装后,还没有安装好,就在aviable中选择下面插件再进行安装。如下:

4.配置jenkins和kubernetes集成:(配置jenkins能连接kubernetes集群)

点击系统管理——系统设置——拉到最底部,有一个cloud,点击add a new cloud,选择kubernetes,如下:

如下:配置kubernetes地址:https://kubernetes.default (dns解析的service名称和命名空间)

配置jenkins地址:http://jenkins.default  (dns解析的service名称和命名空间) 点击save保存

 

5.在jenkins中创建一个测试的pipeline项目实现能代码拉取: 

 点击创建一个流水线任务,如下:

点击ok,如下:

拉到最下面: 

选择相应的项目即可生成对应的pipeline的模板脚本,在模板基础上修改即可。

点击Pipeline Syntax可查看相应的pipeline的相应模块的语法编写,如查看:如何拉取编译代码的语法,点击Pipeline Syntax,如下:

选择git,如下:

[root@k8s-master ~]# cat demo/.git/config  |grep git

url = git@192.168.171.131:/home/git/demo.git

[root@k8s-master ~]# ls /root/.ssh/

id_rsa  id_rsa.pub  known_hosts

[root@k8s-master ~]# ssh-copy-id git@192.168.171.131  #生成秘钥对,将公钥配置在git服务端

[root@k8s-master ~]# cat /root/.ssh/id_rsa   #将私钥添加到jenkins中,实现免交互拉取代码

-----BEGIN RSA PRIVATE KEY-----

MIIEogIBAAKCAQEA1MVh7Da458+hMKQyYit1oc00OCUMwO7r4CIE2RhTd9WkPfu7

9C1is1+FWC/TabVpS3p6Qrt4Up2Wc6Ry/Hx8coKDQkuXEg0fBESNprh7t0n9s4Jf

CfM3P9xDGnIiA9/M4qahu2xq3ZsN9f6/TJUhXL2oOsrq4KPJHH07sT7lp0+08MeM

1c97Rckwd//urQwGC4tdZFTJ9XDVq4S7E0XlKqWQ6NZoorTIj6+w/d1gurZDijQ6

1ry42WDA+JmfmHpQpgmFcc288L/sZgNzeaUWd720Z3KRe5WWM7/YQ8kNrTrBsL7E

SHFbdBcFlKKSdNI19wddRi2lF4AocmN+RuFoPwIDAQABAoIBAEnNHSYd5Q0jC7of

egprIKY60z4WO9JADTj1E9FpiqoqRInrY0pS48VsAIiSLjagYiF4Hig/doDRgHok

NG43j8L4WicIqSHtwzBrxuZpXyA4XfOB6Q0+e49/CxeFNx845YxIWBj2hirPuYv2

DMy3fZb5gfzCna3FoeBr3+Se+HGLpjIUwSXGC/NWHAhYNdqWcHlOSqv8+/sgZ02X

v8BN460ZG+ee+ZUcvrYYbBYOCJgQqY76ihgfLH2MorFvJ5DtGvuYCFlWrbUSxON0

XeKeprP6wXDXtcpVSSdGLvNBIoesCZhVQava0gyNZ0BksUR+5yJQYgH+gEbRP6MA

rjEumiECgYEA8DTolQvNS+Kgb6gVYslmm0LX9ju1izboHQOqoQGaWtG+0NvWXkHQ

pZ1n2JM9z3l8f8gYXog65JO8GgZ5pj5MztFGFbSKzMSwrrLelnKOlBa1iS3607AZ

Lt0Ve4zTRcPXDzvq+wtmtKjNEkArOOlGmjLbU0FQZsbla1tD2SM8Fn0CgYEA4sKv

UTklzC3YiEBCf2zDygPzdaNVO9wroKsbIk/cao5F1RRPcbbnbUlSC+B8iU2N5LyF

njPTRD/0YcaiU+UjmRE6Io8ZgYpWNmCPNBG2gAHJcqxHVOc+vZi2GCM0x/KYV6Nf

lpfwrujYevHzKg11OXX5/RvgmKnepC086XQ5qmsCgYBIdR1HgHTgFC+jHse8sdYC

taohe6dqpBhmUCnTQYGHLcfqhrXmXuIKYy3lFZ1KjXHxtbEsKtnhA3N1QkjFzfsr

U9NZTsmjtR06xXgBV/MMVsFR7Q3Bd1hRW51UVImSfN3NmZxrN+g2TMj3XWXsV+nV

L8wbFBOlvQ+NT81qnYdNnQKBgHjz2WncrrGbHq2GgTHQxRpjQyBTb+pPBSJdQkrq

CCr6GnuOqtV/9y+O9WmKY0Mlj/MbL7Zz9FOqPWNN/VX8miFSVA0NAVDchiHZHWlr

rUnYKii1hhLeidO9uhtj5Apm9yV/6kO8ZdAWOSpTFh4vMYDl0eNVVFZ5/VzbGGt5

6CxjAoGAbZJXbZaIzWjnCb0TvDCb7Ird8M+Ej3eHb6/+EB3YNq01Tv+Z8yOj7sp6

X0GHewSVYXLlmXELL6Ws00s9iYywBksg/AgbFh7K9bboz2oPmHENMQedkf1QuQ9m

j/o/AqYPhjqv5TQ38IrczbPFxiHHK6Zy4yKIX3oZMh+oHAtK5Rk=

-----END RSA PRIVATE KEY-----

 

点击生成脚本,即可得到下面的脚本内容,使用内容填写到pipeline的脚本中即可。

一般也不会出现上面报错,如果上面报错是因为jenkins容器中没有安装git客户端,解决:

[root@k8s-master ~]# kubectl exec -it jjenkin的pod bash

[root@jenkins-0 tomcat]# yum -y install git

脚本内容:git credentialsId: '8ce9e95b-c0a1-4af6-9a86-9b54500745b4', url: 'git@192.168.171.131:/home/git/demo.git'

将脚本内容填写如pipeline的脚本中,实现拉取测试的代码步骤,如下:

node {

   stage('拉取代码') {

      git credentialsId: '8ce9e95b-c0a1-4af6-9a86-9b54500745b4', url: 'git@192.168.171.131:/home/git/demo.git'

      sh 'ls'      

   }

   stage('测试构建') {

      echo 'goujian chenggong'      

   }

}

 点击save,保存后,点击立即构建,如下:

 

 

 

6.在jenkins中创建一个pipeline项目实现CICD:(只做到了CI,CD未做,因为部署到k8s时连接k8s的kubeconfig凭据id不知道怎么添加认证)

1)上传java-demo代码:

[root@k8s-master ~]# ls java-demo.tar.gz

java-demo.tar.gz

[root@k8s-master ~]# tar -zxf java-demo.tar.gz

[root@k8s-master ~]# cd java-demo

[root@k8s-master java-demo]# ls

db  deploy.yml  LICENSE  pom.xml  README.md  src

[root@k8s-master java-demo]# cat src/main/resources/application.yml

server:

  port: 8080

spring:

  datasource:

    url: jdbc:mysql://192.168.171.128:3306/test?characterEncoding=utf-8

    username: demo

    password: 123456

    driver-class-name: com.mysql.jdbc.Driver

  freemarker:

    allow-request-override: false

    cache: true

    check-template-location: true

    charset: UTF-8

    content-type: text/html; charset=utf-8

    expose-request-attributes: false

    expose-session-attributes: false

    expose-spring-macro-helpers: false

    suffix: .ftl

    template-loader-path:

      - classpath:/templates/

[root@k8s-master java-demo]# ls db/

tables_ly_tomcat.sql

2)master上安装mysql并导入sql文件:

[root@k8s-master ~]# yum -y install mariadb mariadb-server

[root@k8s-master ~]# systemctl start mariadb

[root@k8s-master ~]# systemctl enable mariadb

[root@k8s-master ~]# mysql   安装启动mariadb后,不需密码即可登录

MariaDB [(none)]> show databases;

+--------------------+

| Database           |

+--------------------+

| information_schema |

| mysql              |

| performance_schema |

| test               |

+--------------------+

MariaDB [(none)]> quit

接下来进行MariaDB的相关简单配置,使用命令: mysql_secure_installation

[root@k8s-master ~]# mysql_secure_installation   #简单安全安装配置,回车

NOTE: RUNNING ALL PARTS OF THIS SCRIPT IS RECOMMENDED FOR ALL MariaDB

      SERVERS IN PRODUCTION USE!  PLEASE READ EACH STEP CAREFULLY!

In order to log into MariaDB to secure it, we'll need the current

password for the root user.  If you've just installed MariaDB, and

you haven't set the root password yet, the password will be blank,

so you should just press enter here.

Enter current password for root (enter for none):    回车,输入当前密码,无,直接回车即可

OK, successfully used password, moving on...

Setting the root password ensures that nobody can log into the MariaDB

root user without the proper authorisation.  

Set root password? [Y/n] y

New password:             输入要设置的root密码: 123

Re-enter new password:    确认root的密码: 123

Password updated successfully!

Reloading privilege tables..

 ... Success!

By default, a MariaDB installation has an anonymous user, allowing anyone

to log into MariaDB without having to have a user account created for

them.  This is intended only for testing, and to make the installation

go a bit smoother.  You should remove them before moving into a

production environment.   #下面一路都是y

Remove anonymous users? [Y/n] y  是否删除匿名用户,回车

 ... Success!

Normally, root should only be allowed to connect from 'localhost'.  This

ensures that someone cannot guess at the root password from the network.

Disallow root login remotely? [Y/n] y  是否禁止root远程登录,回车

 ... Success!

By default, MariaDB comes with a database named 'test' that anyone can

access.  This is also intended only for testing, and should be removed

before moving into a production environment.

Remove test database and access to it? [Y/n] y  是否删除test数据库,回车

 - Dropping test database...

 ... Success!

 - Removing privileges on test database...

 ... Success!

Reloading the privilege tables will ensure that all changes made so far

will take effect immediately.

Reload privilege tables now? [Y/n] y  是否重新加载权限表,回车

 ... Success!

Cleaning up...

All done!  If you've completed all of the above steps, your MariaDB

installation should now be secure.

Thanks for using MariaDB!

[root@k8s-master ~]# mysql -uroot -p123  #使用设置的密码登录mysql

MariaDB [(none)]> show databases;

+--------------------+

| Database           |

+--------------------+

| information_schema |

| mysql              |

| performance_schema |

+--------------------+

MariaDB [(none)]> create database test;

MariaDB [(none)]> show databases;

+--------------------+

| Database           |

+--------------------+

| information_schema |

| mysql              |

| performance_schema |

| test               |

+--------------------+

MariaDB [(none)]> source /root/java-demo/db/tables_ly_tomcat.sql;

MariaMariaDB [test]> show tables;

+----------------+

| Tables_in_test |

+----------------+

| user           |

+----------------+

MariaDB [(none)]> grant all on test.* to demo@'%' identified by '123456';    #再授权一个能远程连接mysql的root密码

3)将项目代码提交的git仓库中:

在git机器上创建项目仓库:

[root@localhost ~]# su - git

[git@localhost ~]$ mkdir java-demo.git

[git@localhost ~]$ cd java-demo.git/

[git@localhost java-demo.git]$ ls

[git@localhost java-demo.git]$ git --bare init

Initialized empty Git repository in /home/git/java-demo.git/

[git@localhost java-demo.git]$ ls

branches  config  description  HEAD  hooks  info  objects  refs

将master上上传的项目代码提交的git仓库中:

[root@k8s-master ~]# cd java-demo

[root@k8s-master java-demo]# cat .git/config

[core]

repositoryformatversion = 0

filemode = true

bare = false

logallrefupdates = true

[remote "origin"]

url = git@192.168.171.131:/home/git/java-demo.git

fetch = +refs/heads/*:refs/remotes/origin/*

[branch "master"]

remote = origin

merge = refs/heads/master

[root@k8s-master java-demo]# ls

db  deploy.yml  LICENSE  pom.xml  README.md  src

[root@k8s-master java-demo]# git add .

[root@k8s-master java-demo]# git commit -m "test"

[root@k8s-master java-demo]# git remote -v

origin git@192.168.171.131:/home/git/java-demo.git (fetch)

origin git@192.168.171.131:/home/git/java-demo.git (push)

[root@k8s-master java-demo]# git push origin master

4)配置jenkins连接docker-harbor和githarbor的认证并自动生成一个id值,作为绑定,在pipeliene中用该id做访问认证

(1)Jenkins中生成连接到docker-harbor的id:

Jenkins中保存连接到docker-harbor的用户名和密码到jenkins中,并返回一个id供pipeline使用:

点击凭据——系统——全局凭据,点击添加凭据,

 

 输入docker-harbor的用户名和密码admin/Harbor12345,id一会保存后会自动生成一个id,将该id粘贴到pipeline脚本中作为认证harbor的id

 

 记录下jenkins登录认证docker-harbor的id: f9cd4ec2-138b-44b8-97f9-d28ed495e620

 (2)找到Jenkins中生成连接到git的id:(之前测试拉取git代码时候连接git的凭据id),供pipeline脚本连接git认证使用

 

 

连接git的id为: 8ce9e95b-c0a1-4af6-9a86-9b54500745b4 

(3)将凭据绑定在变量中: pipeline中第3步骤构建时候,将用户名

在pipeline语法中生成:将变量和凭据绑定:

 

 上面解释: 变量username和变量password通过harbor-auth这个凭据来生成,点击生成语法后结果为: password用passwordVariable来保存,username用usernameVariable来保存。

这样就可以通过变量username和password来使用凭据harbor-auth能登录认证docker-harbor了。

4)Jenkins构建任务时,手动设置拉取代码的分支控制:pipeline中分支变量,是通过手动传入的分支控制的

选择pipeline的相关任务,如下:

 

 

 在配置中,点击参数化构建过程,选择字符参数,如下:

 

定义pipeline脚本中的变量Branch和其值master,即可手工传入或选择默认的分支master,再在下面编写pipeline的ci流程脚本:

pipeline的脚本内容如下:

// 公共

def registry = "192.168.171.131"

// 项目

def project = "welcome"

def app_name = "demo"

def image_name = "${registry}/${project}/${app_name}:${BUILD_NUMBER}"

def git_address = "git@192.168.171.131:/home/git/java-demo.git"

// 认证

def secret_name = "registry-pull-secret"

def docker_registry_auth = "f9cd4ec2-138b-44b8-97f9-d28ed495e620"

def git_auth = "8ce9e95b-c0a1-4af6-9a86-9b54500745b4"

def k8s_auth = "e692ef4e-eb69-45a3-b976-0ffe60abaf68"

 node {

      // 第一步

      stage('拉取代码'){

         checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address}"]]])

      }

      // 第二步

      stage('代码编译'){

          sh "mvn clean package -Dmaven.test.skip=true"

          sh "ls"

      }

      // 第三步

      stage('构建镜像'){

          withCredentials([usernamePassword(credentialsId: "${docker_registry_auth}", passwordVariable: 'password', usernameVariable: 'username')]) {

            sh """

              echo '

                FROM lizhenliang/tomcat

                RUN rm -rf /usr/local/tomcat/webapps/*

                ADD target/*.war /usr/local/tomcat/webapps/ROOT.war

              ' > Dockerfile

              ls

              ls target

              docker build -t ${image_name} .

              docker login -u ${username} -p '${password}' ${registry}

              docker push ${image_name}

            """

            }

      }      

      // 第四步

      stage('部署到K8S平台'){

          //模拟部署到k8s,因为k8s的认证kubeconfig和绑定凭据不知道怎么生成和添加,下面暂时用不了

          sh "echo deploy to k8s success"

         // sed -i 's#\$IMAGE_NAME#${image_name}#' deploy.yml

         // sed -i 's#\$SECRET_NAME#${secret_name}#' deploy.yml

         // """

         // kubernetesDeploy configs: 'deploy.yml', kubeconfigId: "${k8s_auth}"

      }      

 }

 点击save,如下:

点击立即构建,如下: 

 选择要部署的分支:master,点击Build,如下:

查看日志:

 

 

 

 

7.将ci阶段构建生成的镜像手动在k8s-master中将应用部署到k8s平台:

1)从上面日志中可以看到:ci生成的镜像是: 192.168.171.131/welcome/demo:2

到docker-harbor仓库中查看:

 

2)使用ci构建的镜像部署项目到k8s平台:

[root@k8s-master ~]# ls java-demo/deploy.yml

java-demo/deploy.yml

[root@k8s-master ~]# vim java-demo/deploy.yml  (无状态,且代码都打包都镜像中了,不需要共享存储)

---

apiVersion: apps/v1

kind: Deployment

metadata:

  name: web

spec:

  replicas: 3

  selector:

    matchLabels:

      app: java-demo

  template:

    metadata:

      labels:

        app: java-demo

    spec:

      imagePullSecrets:

      - name: registry-pull-secret

      #- name: $SECRET_NAME

      containers:

      - name: tomcat

        image: 192.168.171.131/welcome/demo:2

        #image: $IMAGE_NAME

        ports:

        - containerPort: 8080

          name: web

        livenessProbe:

          httpGet:

            path: /

            port: 8080

          initialDelaySeconds: 60

          timeoutSeconds: 5

          failureThreshold: 12

        readinessProbe:

          httpGet:

            path: /

            port: 8080

          initialDelaySeconds: 60

          timeoutSeconds: 5

          failureThreshold: 12

---

apiVersion: v1

kind: Service

metadata:

  name: web

spec:

  type: NodePort

  selector:

    app: java-demo

  ports:

    - protocol: TCP

      port: 80

      targetPort: 8080

      nodePort: 30009

---

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: web

spec:

  rules:

  - host: java.example.com

    http:

      paths:

      - path: /

        backend:

          serviceName: web

          servicePort: 80

上面有一个secret:需要将docker-harbor的用户名和密码以secret方式挂载到里面实现harbor认证

[root@k8s-master ~]# kubectl create secret docker-registry registry-pull-secret --docker-username=admin --docker-password=Harbor12345 --docker-email=shi@163.com --docker-server=192.168.171.131   #指定docker-harbor的地址创建secret

[root@k8s-master ~]# kubectl get secret |grep registry

registry-pull-secret         kubernetes.io/dockerconfigjson        1      15s

[root@k8s-master ~]# kubectl apply -f java-demo/deploy.yml

[root@k8s-master ~]# kubectl get pod,svc -o wide

NAME                                  READY   STATUS    RESTARTS   AGE     IP            NODE        NOMINATED NODE   READINESS GATES

pod/jenkins-master-78577cdf6c-clcdd   1/1     Running   0          4h11m   10.244.1.46   k8s-node2   <none>           <none>

pod/web-9c888f756-5vdjv               1/1     Running   0          2m17s   10.244.0.68   k8s-node1   <none>           <none>

pod/web-9c888f756-82tvd               1/1     Running   0          2m17s   10.244.1.47   k8s-node2   <none>           <none>

pod/web-9c888f756-92npj               1/1     Running   0          2m17s   10.244.1.48   k8s-node2   <none>           <none>

NAME                     TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                          AGE     SELECTOR

service/jenkins-master   NodePort    10.0.0.64    <none>        8080:30006/TCP,50000:32224/TCP   19h     name=jenkins-master

service/kubernetes       ClusterIP   10.0.0.1     <none>        443/TCP                          60d     <none>

service/web              NodePort    10.0.0.223   <none>        80:30009/TCP                     2m17s   app=java-demo

部署好应用后访问:http://192.168.171.129:30009/ 或:http://192.168.171.130:30009/ 

 

 

Logo

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐