参考 : k8s学习(二十九) RocketMq集群(双主双从-同步)_文若书生的专栏-CSDN博客

1.版本及说明

1.1.版本

rocketmq : 4.9.2

java : 8

docker : Docker version 18.09.9, build 039a7df9ba

k8s:

Client Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.9", GitCommit:"a17149e1a189050796ced469dbd78d380f2ed5ef", GitTreeState:"clean", BuildDate:"2020-04-16T11:44:51Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"16", GitVersion:"v1.16.9", GitCommit:"a17149e1a189050796ced469dbd78d380f2ed5ef", GitTreeState:"clean", BuildDate:"2020-04-16T11:36:15Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}

1.2.说明

•  k8s集群的存储为nfs存储,部署需要修改nfs的地址(pv.yaml)
• yaml文件的namespace可能需要修改,当前使用的namespace为test-namespace(最后创建时指定namespace好像会覆盖yaml文件的namespace,未校验)

2.准备镜像

2.1.jdk镜像

docker pull java:8-alpine

2.2.构建rocketmq镜像     

2.2.1.下载rocketmq安装包

网络下载 : https://mirrors.tuna.tsinghua.edu.cn/apache/rocketmq/4.9.2/rocketmq-all-4.9.2-bin-release.zip

百度网盘下载 : https://pan.baidu.com/s/18Fu5bnGydrZK61J0nng0Sw(g7j3)

 2.2.2.设置合适的内存

解压之后修改 rocketmq-4.9.2/bin/runbroker.sh文件(当前是调小)

JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn512m"

2.2.3.构建mq镜像

编写 Dockerfile-nameserver

cat > Dockerfile-rocketmq <<'EOF'
FROM java:8-alpine
 
RUN rm -f /etc/localtime \
&& ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
 
ENV LANG en_US.UTF-8
 
ADD rocketmq-4.9.2 /usr/local/rocketmq-4.9.2
RUN mkdir -p /data/rocketmq/store
 
CMD ["/bin/bash"]
EOF

 编写 Dockerfile-nameserver

cat > Dockerfile-nameserver <<'EOF'
FROM java:8-alpine
 
RUN rm -f /etc/localtime \
&& ln -sv /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone
 
ENV LANG en_US.UTF-8
 
ADD rocketmq-4.9.2 /usr/local/rocketmq-4.9.2
 
WORKDIR /usr/local/rocketmq-4.9.2
CMD ["/usr/local/rocketmq-4.9.2/bin/mqnamesrv","-c","/usr/local/rocketmq-4.9.2/conf/namesrv.properties"]
EOF

构建及push

docker build -f Dockerfile-rocketmq -t 172.16.10.190:80/library/rocketmq:4.9.2 .
docker push 172.16.10.190:80/library/rocketmq:4.9.2

docker build -f Dockerfile-nameserver -t 172.16.10.190:80/library/rocketmq-namesrv:4.9.2 .
docker push 172.16.10.190:80/library/rocketmq-namesrv:4.9.2

3.编写配置文件

pv.yaml

cat > pv.yaml <<'EOF'

apiVersion: v1
kind: PersistentVolume
metadata:
  name:  rockermq-pv
spec:
  accessModes:
    - ReadWriteMany
  capacity:
    storage: 100Gi
  persistentVolumeReclaimPolicy: Retain
  storageClassName: nfs
  nfs:
    path: /nfs/rocketmq
    server: 198.168.1.2

EOF

pvc.yaml

cat > pvc.yaml <<'EOF'

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rockermq-pvc
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 100Gi

EOF

broker-a-s.properties

cat > broker-a-s.properties <<'EOF'

brokerClusterName=rocketmq-cluster
brokerName=broker-a
brokerId=1
namesrvAddr=mq-namesrv:20901
defaultTopicQueueNums=4
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
listenPort=20911
deleteWhen=04
fileReservedTime=120
mapedFileSizeCommitLog=1073741824
mapedFileSizeConsumeQueue=300000
diskMaxUsedSpaceRatio=88
storePathRootDir=/nfs/rocketmq/store
maxMessageSize=65536
brokerRole=SLAVE
flushDiskType=SYNC_FLUSH

EOF

broker-a.properties

cat > broker-a.properties <<'EOF'

brokerClusterName=rocketmq-cluster
brokerName=broker-a
brokerId=0
namesrvAddr=mq-namesrv:20901
defaultTopicQueueNums=4
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
listenPort=20911
deleteWhen=04
fileReservedTime=120
mapedFileSizeCommitLog=1073741824
mapedFileSizeConsumeQueue=300000
diskMaxUsedSpaceRatio=88
storePathRootDir=/nfs/rocketmq/store
maxMessageSize=65536
brokerRole=MASTER

EOF

broker-b-s.properties

 

cat > broker-b-s.properties <<'EOF'

brokerClusterName=rocketmq-cluster
brokerName=broker-b
brokerId=1
namesrvAddr=mq-namesrv:20901
defaultTopicQueueNums=4
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
listenPort=20911
deleteWhen=04
fileReservedTime=120
mapedFileSizeCommitLog=1073741824
mapedFileSizeConsumeQueue=300000
diskMaxUsedSpaceRatio=88
storePathRootDir=/nfs/rocketmq/store
maxMessageSize=65536
brokerRole=SLAVE
flushDiskType=SYNC_FLUSH

EOF

broker-b.properties

cat > broker-b.properties <<'EOF'

brokerClusterName=rocketmq-cluster
brokerName=broker-b
brokerId=0
namesrvAddr=mq-namesrv:20901
defaultTopicQueueNums=4
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
listenPort=20911
deleteWhen=04
fileReservedTime=120
mapedFileSizeCommitLog=1073741824
mapedFileSizeConsumeQueue=300000
diskMaxUsedSpaceRatio=88
storePathRootDir=/nfs/rocketmq/store
maxMessageSize=65536
brokerRole=MASTER
flushDiskType=SYNC_FLUSH

EOF

broker-a-s-statefulset.yaml

cat > broker-a-s-statefulset.yaml <<'EOF'

apiVersion: v1
kind: Service
metadata:
  labels:
    app: broker-a-s
  name: broker-a-s
  namespace: test-namespace
spec:
  type: NodePort
  ports:
  - port: 20911
    targetPort: 20911
    name: broker-port
    nodePort: 30912
  selector:
    app: broker-a-s
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: broker-a-s
  namespace: test-namespace
spec:
  serviceName: broker-a-s
  replicas: 1
  selector:
    matchLabels:
      app: broker-a-s
  template:
    metadata:
     labels:
       app: broker-a-s
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - broker-a-s
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
       - name: harborsecret
      containers:
      - name: broker-a-s
        image: registry.gts.com.cn:5000/library/rocketmq:4.9.2
        imagePullPolicy: Always
        command: ["sh","-c","/usr/local/rocketmq-4.9.2/bin/mqbroker  -c /usr/local/rocketmq-4.9.2/conf/broker-a-s.properties"]
        volumeMounts:
          - mountPath: /root/logs
            name: rocketmq-data
            subPath: mq-brokeroptlogs
          - mountPath: /data/rocketmq
            name: rocketmq-data
            subPath: mq-brokeroptstore
          - name: broker-config
            mountPath: /usr/local/rocketmq-4.9.2/conf/broker-a-s.properties
            subPath: broker-a-s.properties
          - name: time
            mountPath: /etc/localtime
        lifecycle:
          postStart:
            exec:
              command: ["/bin/sh","-c","touch /tmp/health"]
        livenessProbe:
          exec:
            command: ["test","-e","/tmp/health"]
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
        readinessProbe:
          tcpSocket:
            port: 20911
          initialDelaySeconds: 15
          timeoutSeconds: 5
          periodSeconds: 20
      volumes:
      - name: rocketmq-data
        persistentVolumeClaim:
          claimName: rockermq-pvc
      - name: broker-config
        configMap:
          name: rocketmq-config
          items:
          - key: broker-a-s.properties
            path: broker-a-s.properties
      - name: time
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

EOF

broker-a-statefulset.yaml

cat > broker-a-statefulset.yaml <<'EOF'

apiVersion: v1
kind: Service
metadata:
  labels:
    app: broker-a
  name: broker-a
  namespace: test-namespace
spec:
  type: NodePort
  ports:
  - port: 20911
    targetPort: 20911
    name: broker-port
    nodePort: 30911
  selector:
    app: broker-a
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: broker-a
  namespace: test-namespace
spec:
  serviceName: broker-a
  replicas: 1
  selector:
    matchLabels:
      app: broker-a
  template:
    metadata:
     labels:
       app: broker-a
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - broker-a
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
       - name: harborsecret
      containers:
      - name: broker-a
        image: registry.gts.com.cn:5000/library/rocketmq:4.9.2
        imagePullPolicy: Always
        command: ["sh","-c","/usr/local/rocketmq-4.9.2/bin/mqbroker  -c /usr/local/rocketmq-4.9.2/conf/broker-a.properties"]
        volumeMounts:
          - mountPath: /root/logs
            name: rocketmq-data
            subPath: mq-brokeroptlogs
          - mountPath: /data/rocketmq
            name: rocketmq-data
            subPath: mq-brokeroptstore
          - name: broker-config
            mountPath: /usr/local/rocketmq-4.9.2/conf/broker-a.properties
            subPath: broker-a.properties
          - name: time
            mountPath: /etc/localtime
        lifecycle:
          postStart:
            exec:
              command: ["/bin/sh","-c","touch /tmp/health"]
        livenessProbe:
          exec:
            command: ["test","-e","/tmp/health"]
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
        readinessProbe:
          tcpSocket:
            port: 20911
          initialDelaySeconds: 15
          timeoutSeconds: 5
          periodSeconds: 20
      volumes:
      - name: rocketmq-data
        persistentVolumeClaim:
          claimName: rockermq-pvc
      - name: broker-config
        configMap:
          name: rocketmq-config
      - name: time
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

EOF

broker-b-s-statefulset.yaml

cat > broker-b-s-statefulset.yaml <<'EOF'

apiVersion: v1
kind: Service
metadata:
  labels:
    app: broker-b-s
  name: broker-b-s
  namespace: test-namespace
spec:
  type: NodePort
  ports:
  - port: 20911
    targetPort: 20911
    name: broker-port
    nodePort: 30914
  selector:
    app: broker-b-s
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: broker-b-s
  namespace: test-namespace
spec:
  serviceName: broker-b-s
  replicas: 1
  selector:
    matchLabels:
      app: broker-b-s
  template:
    metadata:
     labels:
       app: broker-b-s
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - broker-b-s
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
       - name: harborsecret
      containers:
      - name: broker-b-s
        image: registry.gts.com.cn:5000/library/rocketmq:4.9.2
        imagePullPolicy: Always
        command: ["sh","-c","/usr/local/rocketmq-4.9.2/bin/mqbroker  -c /usr/local/rocketmq-4.9.2/conf/broker-b-s.properties"]
        volumeMounts:
          - mountPath: /root/logs
            name: rocketmq-data
            subPath: mq-brokeroptlogs
          - mountPath: /data/rocketmq
            name: rocketmq-data
            subPath: mq-brokeroptstore
          - name: broker-config
            mountPath: /usr/local/rocketmq-4.9.2/conf/broker-b-s.properties
            subPath: broker-b-s.properties
          - name: time
            mountPath: /etc/localtime
        lifecycle:
          postStart:
            exec:
              command: ["/bin/sh","-c","touch /tmp/health"]
        livenessProbe:
          exec:
            command: ["test","-e","/tmp/health"]
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
        readinessProbe:
          tcpSocket:
            port: 20911
          initialDelaySeconds: 15
          timeoutSeconds: 5
          periodSeconds: 20
      volumes:
      - name: rocketmq-data
        persistentVolumeClaim:
          claimName: rockermq-pvc
      - name: broker-config
        configMap:
          name: rocketmq-config
          items:
          - key: broker-b-s.properties
            path: broker-b-s.properties
      - name: time
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

EOF

broker-b-statefulset.yaml

cat > broker-b-statefulset.yaml <<'EOF'

apiVersion: v1
kind: Service
metadata:
  labels:
    app: broker-b
  name: broker-b
  namespace: test-namespace
spec:
  type: NodePort
  ports:
  - port: 20911
    targetPort: 20911
    name: broker-port
    nodePort: 30913
  selector:
    app: broker-b
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: broker-b
  namespace: test-namespace
spec:
  serviceName: broker-b
  replicas: 1
  selector:
    matchLabels:
      app: broker-b
  template:
    metadata:
     labels:
       app: broker-b
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - broker-b
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
       - name: harborsecret
      containers:
      - name: broker-b
        image: registry.gts.com.cn:5000/library/rocketmq:4.9.2
        imagePullPolicy: Always
        command: ["sh","-c","/usr/local/rocketmq-4.9.2/bin/mqbroker  -c /usr/local/rocketmq-4.9.2/conf/broker-b.properties"]
        volumeMounts:
          - mountPath: /root/logs
            name: rocketmq-data
            subPath: mq-brokeroptlogs
          - mountPath: /data/rocketmq
            name: rocketmq-data
            subPath: mq-brokeroptstore
          - name: broker-config
            mountPath: /usr/local/rocketmq-4.9.2/conf/broker-b.properties
            subPath: broker-b.properties
          - name: time
            mountPath: /etc/localtime
        lifecycle:
          postStart:
            exec:
              command: ["/bin/sh","-c","touch /tmp/health"]
        livenessProbe:
          exec:
            command: ["test","-e","/tmp/health"]
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
        readinessProbe:
          tcpSocket:
            port: 20911
          initialDelaySeconds: 15
          timeoutSeconds: 5
          periodSeconds: 20
      volumes:
      - name: rocketmq-data
        persistentVolumeClaim:
          claimName: rockermq-pvc
      - name: broker-config
        configMap:
          name: rocketmq-config
      - name: time
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

EOF

namesrv-statefulset.yaml

cat > namesrv-statefulset.yaml <<'EOF'

apiVersion: v1
kind: Service
metadata:
  labels:
    app: mq-namesrv
  name: mq-namesrv
  namespace: test-namespace
spec:
  type: NodePort
  ports:
  - port: 20901
    targetPort: 20901
    name: namesrv-port
    nodePort: 30915
  selector:
    app: mq-namesrv
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: mq-namesrv
  namespace: test-namespace
spec:
  serviceName: mq-namesrv
  replicas: 1
  selector:
    matchLabels:
      app: mq-namesrv
  template:
    metadata:
     labels:
       app: mq-namesrv
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values:
                      - mq-namesrv
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
       - name: harborsecret
      containers:
      - name: mq-namesrv
        image: registry.gts.com.cn:5000/library/rocketmq-namesrv:4.9.2
        imagePullPolicy: Always
        volumeMounts:
          - name: time
            mountPath: /etc/localtime
        lifecycle:
          postStart:
            exec:
              command: ["/bin/sh","-c","touch /tmp/health"]
        livenessProbe:
          exec:
            command: ["test","-e","/tmp/health"]
          initialDelaySeconds: 5
          timeoutSeconds: 5
          periodSeconds: 10
        readinessProbe:
          tcpSocket:
            port: 20901
          initialDelaySeconds: 15
          timeoutSeconds: 5
          periodSeconds: 20
      volumes:
      - name: time
        hostPath:
          path: /usr/share/zoneinfo/Asia/Shanghai

EOF

4.创建

kubectl create configmap rocketmq-config --from-file=broker-a.properties --from-file=broker-b.properties --from-file=broker-a-s.properties --from-file=broker-b-s.properties  -n test-namespace
kubectl apply -f broker-a-statefulset.yaml -n test-namespace
kubectl apply -f broker-a-s-statefulset.yaml -n test-namespace
kubectl apply -f broker-b-statefulset.yaml -n test-namespace
kubectl apply -f broker-b-s-statefulset.yaml -n test-namespace
kubectl apply -f namesrv-statefulset.yaml -n test-namespace

5.删除

kubectl delete -f broker-a-statefulset.yaml -n test-namespace
kubectl delete -f broker-a-s-statefulset.yaml -n test-namespace
kubectl delete -f broker-b-statefulset.yaml -n test-namespace
kubectl delete -f broker-b-s-statefulset.yaml -n test-namespace
kubectl delete -f namesrv-statefulset.yaml -n test-namespace
kubectl delete configmap rocketmq-config -n test-namespace

6.完成

mq k8s内部地址为 : mq-namesrv:20901

mq 外部地址为 : 任意masterip:30915