dev环境k8s集群部署Apollo

基本信息

apollo版本: 1.7.1
数据库使用阿里云rds，版本为MySQL5.6
数据库用户: apollo db_passwd
数据库连接地址: rm-xxxxxxxxxxxxxxxxx.mysql.rds.aliyuncs.com

部署

下载1.7.1安装包

下载地址

ops@dev-jump:/tmp/apollo$ pwd
/tmp/apollo
ops@dev-jump:/tmp/apollo$ ls
apollo-1.7.1.zip  apollo-adminservice-1.7.1-github.zip  apollo-configservice-1.7.1-github.zip  apollo-portal-1.7.1-github.zip

处理安装包

# 创建部署目录
ops@dev-jump:/tmp/apollo$ mkdir ./deploy
ops@dev-jump:/tmp/apollo$ unzip apollo-1.7.1.zip
ops@dev-jump:/tmp/apollo$ unzip -o apollo-configservice-1.7.1-github.zip  -d ./deploy/apollo-config-server/
ops@dev-jump:/tmp/apollo$ unzip -o apollo-adminservice-1.7.1-github.zip -d ./deploy/apollo-admin-server/
ops@dev-jump:/tmp/apollo$ unzip -o apollo-portal-1.7.1-github.zip -d ./deploy/apollo-portal-server/

# 将部署所需文件移动至部署目录
ops@dev-jump:/tmp/apollo$ cp -R ./apollo-1.7.1/scripts/apollo-on-kubernetes/* ./deploy
ops@dev-jump:/tmp/apollo$ mv ./deploy/apollo-config-server/apollo-configservice-1.7.1.jar ./deploy/apollo-config-server/apollo-configservice.jar
ops@dev-jump:/tmp/apollo$ mv ./deploy/apollo-admin-server/apollo-adminservice-1.7.1.jar ./deploy/apollo-admin-server/apollo-adminservice.jar
ops@dev-jump:/tmp/apollo$ mv ./deploy/apollo-portal-server/apollo-portal-1.7.1.jar  ./deploy/apollo-portal-server/apollo-portal.jar

# 构建镜像
ops@dev-jump:/tmp/apollo$ docker build -t registry-vpc.cn-beijing.aliyuncs.com/xxx/alpine-bash:3.8 ./deploy/alpine-bash-3.8-image
ops@dev-jump:/tmp/apollo$ docker build -t registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-config-server:v1.7.1 ./deploy/apollo-config-server
ops@dev-jump:/tmp/apollo$ docker build -t registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-admin-server:v1.7.1 ./deploy/apollo-admin-server
ops@dev-jump:/tmp/apollo$ docker build -t registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-portal-server:v1.7.1 ./deploy/apollo-portal-server

# 推送镜像到镜像仓库
ops@dev-jump:/tmp/apollo$ docker push registry-vpc.cn-beijing.aliyuncs.com/xxx/alpine-bash:3.8
ops@dev-jump:/tmp/apollo$ docker push registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-config-server:v1.7.1
ops@dev-jump:/tmp/apollo$ docker push registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-admin-server:v1.7.1
ops@dev-jump:/tmp/apollo$ docker push registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-portal-server:v1.7.1

创建数据库

ops@dev-jump:/tmp/apollo$ ls ./deploy/db/
config-db-dev  config-db-prod  config-db-test-alpha  config-db-test-beta  portal-db

ops@dev-jump:/tmp/apollo$ ls ./deploy/db/config-db-dev/
apolloconfigdb.sql

ops@dev-jump:/tmp/apollo$ ls ./deploy/db/portal-db/
apolloportaldb.sql

打开上面两个sql文件，可以看到apollo需要创建两个数据库apolloportaldb、devapolloconfigdb
分别在rds控制台的"账号管理"和"数据库管理"上创建apollo用户和两个数据库，注意给apollo分配这两个数据库的读写权限
然后在dms控制台导入sql文件

部署apollo资源

apollo相关资源将部署在ops命名空间下

ops@dev-jump:/tmp/apollo$ ls ./deploy/kubernetes/apollo-env-dev/
service-apollo-admin-server-dev.yaml  service-apollo-config-server-dev.yaml  service-mysql-for-apollo-dev-env.yaml

apollo-config-server

修改安装包给的默认yaml文件

ops@dev-jump:/tmp/apollo$ vim ./deploy/kubernetes/apollo-env-dev/service-apollo-config-server-dev.yaml
# 修改资源名称，默认的资源名称太长了
# 所有资源的namespace改成ops
# 数据库连接地址、用户、用户名密码
# 因为只保留一个副本，故仅保留ConfigMap的第一个eureka.service.url
# 因修改了资源名称，故eureka.service.url也需要按照资源名称进行修改
# 修改label、selector，默认的太长了
# 同时StatfulSet副本保留1个
# 修改默认的镜像为咱们自己构建的镜像
# 增加imagePullSecrets，这是集群拉取镜像所使用的的secret
# 增加一个volume，挂载时区
# 修改APOLLO_CONFIG_SERVICE_NAME 环境变量的值

修改完如下

---
# configmap for apollo-config-server-dev
kind: ConfigMap
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-config-server
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://rm-xxxxxxxxxxxxxxxxx.mysql.rds.aliyuncs.com:3306/DevApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = apollo
    spring.datasource.password = db_passwd
    eureka.service.url = http://apollo-config-server-0.apollo-meta-server:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-meta-server
  labels:
    app: apollo-meta-server
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
  selector:
    app: apollo-config-server
  type: ClusterIP
  clusterIP: None
  sessionAffinity: ClientIP

---
kind: Service
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-config-server
  labels:
    app: apollo-config-server
spec:
  ports:
    - protocol: TCP
      port: 8080
      targetPort: 8080
      nodePort: 30002
  selector:
    app: apollo-config-server
  type: NodePort
  sessionAffinity: ClientIP

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  namespace: ops
  name: apollo-config-server
  labels:
    app: apollo-config-server
spec:
  serviceName: apollo-meta-server
  replicas: 1
  selector:
    matchLabels:
      app: apollo-config-server
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: apollo-config-server
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - apollo-config-server
              topologyKey: kubernetes.io/hostname

      volumes:
        - name: volume-configmap-apollo-config-server-dev
          configMap:
            name: apollo-config-server
            items:
              - key: application-github.properties
                path: application-github.properties
        - hostPath:
            path: /usr/share/zoneinfo/Asia/Shanghai
            type: ""
          name: timezone
          # 增加时区
        
        
      containers:
        - image: registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-config-server:v1.7.1
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: apollo-config-server
          ports:
            - protocol: TCP
              containerPort: 8080

          volumeMounts:
            - name: volume-configmap-apollo-config-server-dev
              mountPath: /apollo-config-server/config/application-github.properties
              subPath: application-github.properties
            - mountPath: /etc/localtime
              name: timezone

          env:
            - name: APOLLO_CONFIG_SERVICE_NAME
              value: "apollo-config-server.ops"
              # 注意修改namespace

          readinessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds: 10
            periodSeconds: 5

          livenessProbe:
            tcpSocket:
              port: 8080
            initialDelaySeconds:  120
            periodSeconds: 10

      dnsPolicy: ClusterFirst
      restartPolicy: Always
      imagePullSecrets:
      - name: regsecret

# 若未创建ns，则创建
ops@dev-jump:/tmp/apollo$ kubectl create namespace ops
ops@dev-jump:/tmp/apollo$ kubectl create -f ./deploy/kubernetes/apollo-env-dev/service-apollo-config-server-dev.yaml

观察apollo-config-server的Pod，若已正常启动，则可进入下一步

ops@dev-jump:/tmp/apollo$ watch 'kubectl get -f deploy/kubernetes/apollo-env-dev/service-apollo-config-server-dev.yaml -o wide '

apollo-admin-server

ops@dev-jump:/tmp/apollo$ vim  deploy/kubernetes/apollo-env-dev/service-apollo-admin-server-dev.yaml
# 修改的地方和apollo-config-server类似
# 修改initContainers启动命令中的url

修改完的yaml文件如下

---
kind: ConfigMap
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-admin-server
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://rm-xxxxxxxxxxxxxxxxx.mysql.rds.aliyuncs.com:3306/DevApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = apollo
    spring.datasource.password = db_passwd
    eureka.service.url = http://apollo-config-server-0.apollo-meta-server:8080/eureka/

---
kind: Service
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-admin-server
  labels:
    app: apollo-admin-server
spec:
  ports:
    - protocol: TCP
      port: 8090
      targetPort: 8090
  selector:
    app: apollo-admin-server
  type: ClusterIP
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: ops
  name: apollo-admin-server
  labels:
    app: apollo-admin-server
spec:
  replicas: 1
  selector:
    matchLabels:
      app: apollo-admin-server
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: apollo-admin-server
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - apollo-admin-server
              topologyKey: kubernetes.io/hostname

      volumes:
        - name: volume-configmap-apollo-admin-server-dev
          configMap:
            name: apollo-admin-server
            items:
              - key: application-github.properties
                path: application-github.properties

      initContainers:
        - image: registry-vpc.cn-beijing.aliyuncs.com/xxx/alpine-bash:3.8
          name: check-service-apollo-config-server-dev
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 apollo-config-server.ops:8080"]

      containers:
        - image: registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-admin-server:v1.7.1
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: apollo-admin-server
          ports:
            - protocol: TCP
              containerPort: 8090

          volumeMounts:
            - name: volume-configmap-apollo-admin-server-dev
              mountPath: /apollo-admin-server/config/application-github.properties
              subPath: application-github.properties

          env:
            - name: APOLLO_ADMIN_SERVICE_NAME
              value: "apollo-admin-server.ops"

          readinessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 10
            periodSeconds: 5

          livenessProbe:
            tcpSocket:
              port: 8090
            initialDelaySeconds: 120
            periodSeconds: 10

      dnsPolicy: ClusterFirst
      restartPolicy: Always
      imagePullSecrets:
      - name: regsecret

部署

ops@dev-jump:/tmp/apollo$ kubectl create -f deploy/kubernetes/apollo-env-dev/service-apollo-admin-server-dev.yaml

apollo-portal-server

ops@dev-jump:/tmp/apollo$ vim deploy/kubernetes/service-apollo-portal-server.yaml
# 修改命名空间
# 修改资源名称
# 去掉mysql svc和ep
# 修改数据库连接地址、用户密码
# 删除多余的apollo-env.properties，仅保留dev环境，并修改url
# 修改initContainers启动命令中的url，并去掉其他环境命令
# 修改APOLLO_PORTAL_SERVICE_NAME环境变量的值和命名空间

修改完的yaml文件如下

---
kind: ConfigMap
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-portal-server
data:
  application-github.properties: |
    spring.datasource.url = jdbc:mysql://rm-xxxxxxxxxxxxxxxxx.mysql.rds.aliyuncs.com:3306/ApolloPortalDB?characterEncoding=utf8
    spring.datasource.username = apollo
    spring.datasource.password = db_passwd
  apollo-env.properties: |
    dev.meta=http://apollo-config-server.ops:8080

---
kind: Service
apiVersion: v1
metadata:
  namespace: ops
  name: apollo-portal-server
  labels:
    app: apollo-portal-server
spec:
  ports:
    - protocol: TCP
      port: 8070
      targetPort: 8070
      nodePort: 30001
  selector:
    app: apollo-portal-server
  type: NodePort
  sessionAffinity: ClientIP

---
kind: Deployment
apiVersion: apps/v1
metadata:
  namespace: ops
  name: apollo-portal-server
  labels:
    app: apollo-portal-server
spec:
  replicas: 1
  selector:
    matchLabels:
      app: apollo-portal-server
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: apollo-portal-server
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - apollo-portal-server
              topologyKey: kubernetes.io/hostname
      volumes:
        - name: volume-configmap-apollo-portal-server
          configMap:
            name: apollo-portal-server
            items:
              - key: application-github.properties
                path: application-github.properties
              - key: apollo-env.properties
                path: apollo-env.properties

      initContainers:
        - image: registry-vpc.cn-beijing.aliyuncs.com/xxx/alpine-bash:3.8
          name: check-service-apollo-admin-server-dev
          command: ['bash', '-c', "curl --connect-timeout 2 --max-time 5 --retry 60 --retry-delay 1 --retry-max-time 120 apollo-admin-server.ops:8090"]

      containers:
        - image: registry-vpc.cn-beijing.aliyuncs.com/xxx/apollo-portal-server:v1.7.1
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          name: apollo-portal-server
          ports:
            - protocol: TCP
              containerPort: 8070

          volumeMounts:
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/application-github.properties
              subPath: application-github.properties
            - name: volume-configmap-apollo-portal-server
              mountPath: /apollo-portal-server/config/apollo-env.properties
              subPath: apollo-env.properties

          env:
            - name: APOLLO_PORTAL_SERVICE_NAME
              value: "apollo-portal-server.ops"

          readinessProbe:
            tcpSocket:
              port: 8070
            initialDelaySeconds: 10
            periodSeconds: 5

          livenessProbe:
            tcpSocket:
              port: 8070
            initialDelaySeconds: 120
            periodSeconds: 15

      dnsPolicy: ClusterFirst
      restartPolicy: Always
      imagePullSecrets:
      - name: regsecret

部署

ops@dev-jump:/tmp/apollo$ kubectl create -f deploy/kubernetes/service-apollo-portal-server.yaml

配置访问

使用阿里云内网slb，负载均衡到k8s集群的30001NodePort上
slb实例ip: 172.x.x.x
配置tcp监听，前端端口为30001，后端服务器为开发环境k8s-node，端口30001(apollo-portal-server的svc的NodePort端口)
本地浏览器访问172.x.x.x:30001，便可以看到apollo-portal界面

默认账号
apollo admin

修改密码为 my-apollo

参考文章