二进制k8s集群部署gitlab

环境准备

##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.54  nfs

## keepalived的vip
192.168.48.66

IP	Hostname	CPU	Memory
192.168.48.101	master01	2	4G
192.168.48.102	master02	2	4G
192.168.48.103	master03	2	4G
192.168.48.201	node01	2	4G
192.168.48.202	node02	2	4G
192.168.48.54	nfs	2	4G

软件	版本
kubernetes	1.15.2
docker-ce	19.03
calico	3.8
etcd	3.3.13
CNI	0.8.1
coredns	1.4.0
metrics-server	0.3.3
ingress-controller	0.25.0
dashboard	1.10.1
Weave Scope	1.11.4
nfs	v4
helm	2.14.3
harbor	1.1.1
gitlab	1.11.8

编写gitlab-redis.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: redis-pvc
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "k8s-nfs-storage"
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: redis
  namespace: kube-ops
  labels:
    name: redis
spec:
  template:
    metadata:
      name: redis
      labels:
        name: redis
    spec:
      containers:
      - name: redis
        image: sameersbn/redis
        imagePullPolicy: IfNotPresent
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - mountPath: /var/lib/redis
          name: data
        livenessProbe:
          exec:
            command:
            - redis-cli
            - ping
          initialDelaySeconds: 30
          timeoutSeconds: 5
        readinessProbe:
          exec:
            command:
            - redis-cli
            - ping
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: redis-pvc

---
apiVersion: v1
kind: Service
metadata:
  name: redis
  namespace: kube-ops
  labels:
    name: redis
spec:
  ports:
    - name: redis
      port: 6379
      targetPort: redis
  selector:
    name: redis

编写gitlab-postgresql.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: postgresql-pvc
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "k8s-nfs-storage"
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: postgresql
  namespace: kube-ops
  labels:
    name: postgresql
spec:
  template:
    metadata:
      name: postgresql
      labels:
        name: postgresql
    spec:
      containers:
      - name: postgresql
        image: sameersbn/postgresql:10
        imagePullPolicy: IfNotPresent
        env:
        - name: DB_USER
          value: gitlab
        - name: DB_PASS
          value: passw0rd
        - name: DB_NAME
          value: gitlab_production
        - name: DB_EXTENSION
          value: pg_trgm
        ports:
        - name: postgres
          containerPort: 5432
        volumeMounts:
        - mountPath: /var/lib/postgresql
          name: data
        livenessProbe:
          exec:
            command:
            - pg_isready
            - -h
            - localhost
            - -U
            - postgres
          initialDelaySeconds: 30
          timeoutSeconds: 5
        readinessProbe:
          exec:
            command:
            - pg_isready
            - -h
            - localhost
            - -U
            - postgres
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: postgresql-pvc

---
apiVersion: v1
kind: Service
metadata:
  name: postgresql
  namespace: kube-ops
  labels:
    name: postgresql
spec:
  ports:
    - name: postgres
      port: 5432
      targetPort: postgres
  selector:
    name: postgresql

编写gitlab.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: gitlab-pvc
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "k8s-nfs-storage"
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: gitlab
  namespace: kube-ops
  labels:
    name: gitlab
spec:
  template:
    metadata:
      name: gitlab
      labels:
        name: gitlab
    spec:
      containers:
      - name: gitlab
        image: sameersbn/gitlab:11.8.1
        imagePullPolicy: IfNotPresent
        env:
        - name: TZ
          value: Asia/Shanghai
        - name: GITLAB_TIMEZONE
          value: Beijing
        - name: GITLAB_SECRETS_DB_KEY_BASE
          value: long-and-random-alpha-numeric-string
        - name: GITLAB_SECRETS_SECRET_KEY_BASE
          value: long-and-random-alpha-numeric-string
        - name: GITLAB_SECRETS_OTP_KEY_BASE
          value: long-and-random-alpha-numeric-string
        - name: GITLAB_ROOT_PASSWORD
          value: admin321
        - name: GITLAB_ROOT_EMAIL
          value: 1247549534@qq.com
        - name: GITLAB_HOST
          value: gitlab.tk8s.com
        - name: GITLAB_PORT
          value: "80"
        - name: GITLAB_SSH_PORT
          value: "22"
        - name: GITLAB_NOTIFY_ON_BROKEN_BUILDS
          value: "true"
        - name: GITLAB_NOTIFY_PUSHER
          value: "false"
        - name: GITLAB_BACKUP_SCHEDULE
          value: daily
        - name: GITLAB_BACKUP_TIME
          value: 01:00
        - name: DB_TYPE
          value: postgres
        - name: DB_HOST
          value: postgresql
        - name: DB_PORT
          value: "5432"
        - name: DB_USER
          value: gitlab
        - name: DB_PASS
          value: passw0rd
        - name: DB_NAME
          value: gitlab_production
        - name: REDIS_HOST
          value: redis
        - name: REDIS_PORT
          value: "6379"
        ports:
        - name: http
          containerPort: 80
        - name: ssh
          containerPort: 22
        volumeMounts:
        - mountPath: /home/git/data
          name: data
        livenessProbe:
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 180
          timeoutSeconds: 5
        readinessProbe:
          httpGet:
            path: /
            port: 80
          initialDelaySeconds: 5
          timeoutSeconds: 1
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: gitlab-pvc

---
apiVersion: v1
kind: Service
metadata:
  name: gitlab
  namespace: kube-ops
  labels:
    name: gitlab
spec:
  ports:
    - name: http
      port: 80
      targetPort: http
    - name: ssh
      port: 22
      targetPort: ssh
      nodePort: 30022
  type: NodePort
  selector:
    name: gitlab

---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: gitlab
  namespace: kube-ops
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - host: gitlab.tk8s.com
    http:
      paths:
      - backend:
          serviceName: gitlab
          servicePort: http

镜像下载

sameersbn/gitlab:11.8.1
sameersbn/postgresql:10
sameersbn/redis

下载地址

链接: https://pan.baidu.com/s/1JdH2kA3xtOW527jtbfntIA 提取码: dquf

docker load -i gitlab.tar.gz

安装

[root@master01 gitlab]# kubectl apply -f gitlab-redis.yaml
persistentvolumeclaim/redis-pvc created
deployment.apps/redis created
service/redis created

[root@master01 gitlab]# kubectl apply -f gitlab-postgresql.yaml 
persistentvolumeclaim/postgresql-pvc created
deployment.apps/postgresql created
service/postgresql created

[root@master01 gitlab]# kubectl apply -f gitlab.yaml 
persistentvolumeclaim/gitlab-pvc created
deployment.apps/gitlab created
service/gitlab created
ingress.extensions/gitlab created

访问

测试

建立一个组

建立一个用户

设置用户密码

将用户添加到组中

用创建的用户登录，建立一个项目test

添加ssh-key

[root@master01 ~]#  ssh-keygen -t rsa -C "tk8s@tk8s.com"
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
/root/.ssh/id_rsa already exists.
Overwrite (y/n)? y
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:EOiVSljHIBauPckDFQ1C1zlpjEWSHdILGUsW291O8Ps tk8s@tk8s.com
The key's randomart image is:
+---[RSA 2048]----+
|o.B//B*o         |
| ==*X@=.+        |
|. .=o++. +       |
| = .o. .o .      |
|. *     So       |
|   o      .      |
|           E     |
|                 |
|                 |
+----[SHA256]-----+
[root@master01 ~]# cat .ssh/id_rsa.pub 
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaV/YZqafXM9Uxepi7DF2fifcFK+Zvt4XggJ5AoY+aUfNVt0fPor14Oum0oWEVyozTLPPo9Q9ppaudfoHzrgZxKyDn4Qs/sEjPnE97iu3eJGv2WdUk16UVvDuHIYR26klAt6dcka2w8HbJJCo25r9rVPWjISAWnm67fFG1KQMrOK8BTEKfYbw4LM3WwgG5NRHiOhBoEfbAXMmX1ue1gAOjFMIOncAZQb69TZQ4Sg7TdPccCDH/rZsAOavFvcNn5N5Z7txZ4f4WN2AmUvbCS14KypK5aY3PnaLwtCgZR/yQJhaP0K6zFvj0E7kgDzcQSytAoLZX0rUtazteoaixtHBv tk8s@tk8s.com

查看gitlab的svc ，查看22端口的nodeport

[root@master01 ~]#  kubectl get svc -n kube-ops
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                          AGE
gitlab       NodePort    10.108.242.180   <none>        80:30385/TCP,22:30022/TCP        14m
jenkins      NodePort    10.98.175.119    <none>        8080:32002/TCP,50000:32233/TCP   18h
postgresql   ClusterIP   10.111.115.149   <none>        5432/TCP                         14m
redis        ClusterIP   10.102.248.110   <none>        6379/TCP                         14m

下载test项目

[root@master01 ~]# git config --global user.name "tk8s"
[root@master01 ~]# git config --global user.email "tk8s@tk8s.com"

[root@master01 ~]# git clone ssh://git@gitlab.tk8s.com:30022/test/test.git
Cloning into 'test'...
The authenticity of host '[gitlab.tk8s.com]:30022 ([192.168.48.101]:30022)' can't be established.
ECDSA key fingerprint is SHA256:BVVyBuuTxg+VkHRLre0elDKfbntzUZGq1GtNINXidUI.
ECDSA key fingerprint is MD5:69:fa:b3:5b:c0:81:98:88:17:06:9f:6f:8e:bf:3c:d4.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '[gitlab.tk8s.com]:30022,[192.168.48.101]:30022' (ECDSA) to the list of known hosts.
warning: You appear to have cloned an empty repository.

[root@master01 ~]# cd test
[root@master01 test]# echo test > test.txt
[root@master01 test]# git add .
[root@master01 test]# git commit -m "test"
[master (root-commit) 8c08e21] test
 1 file changed, 1 insertion(+)
 create mode 100644 test.txt
[root@master01 test]# git push origin master
Counting objects: 3, done.
Writing objects: 100% (3/3), 199 bytes | 0 bytes/s, done.
Total 3 (delta 0), reused 0 (delta 0)
To ssh://git@gitlab.tk8s.com:30022/test/test.git
 * [new branch]      master -> master