k8s部署rabbitmq集群（内网环境）
·
固定nfs动态存储用到的端口
sudo vi /etc/sysconfig/nfs
# 固定 statd 端口（UDP/TCP 统一为32769）
STATD_PORT=32769
# 固定 nlockmgr 端口（UDP/TCP 统一为32803）
LOCKD_TCP_PORT=32803
LOCKD_UDP_PORT=32803
# 固定pvc配额
RQUOTAD_PORT=875
重启使配置生效
# 停止相关服务
sudo systemctl stop nfs-server rpc-statd rpcbind
# 启动服务（顺序：先rpcbind，再statd，最后nfs-server）
sudo systemctl start rpcbind rpc-statd nfs-server
# 重启后验证服务状态（确保都是running）
sudo systemctl status rpcbind rpc-statd nfs-server
验证端口是否固定成功
# 1. 重新查询rpc注册的端口（核心验证）
sudo rpcinfo -p localhost
# 2. 单独验证statd端口（32769）
sudo ss -tulnp | grep 32769
# 3. 单独验证nlockmgr端口（32803）
sudo ss -tulnp | grep 32803
创建命名空间
kubectl create namespace rq
确认 ServiceAccount（确保 RabbitMQ 能访问 K8s API）
# 检查ServiceAccount是否存在
kubectl get sa rabbitmq-cluster -n rq
# 若不存在，创建并绑定权限
kubectl create sa rabbitmq-cluster -n rq
kubectl create clusterrolebinding rabbitmq-cluster --clusterrole=view --serviceaccount=rq:rabbitmq-cluster
确认 StorageClass（确保 NFS 动态存储可用）
kubectl get sc nfs-client # 需存在NFS类型的StorageClass
编写rabbitmq镜像Dockerfile(事先准备低延迟组件rabbitmq_delayed_message_exchange-3.13.0.ez)
FROM rabbitmq:3.13.3-management
LABEL maintainer="Lion Li"
ADD ./rabbitmq_delayed_message_exchange-3.13.0.ez /plugins
RUN rabbitmq-plugins enable rabbitmq_delayed_message_exchange
ENTRYPOINT ["rabbitmq-server"]
部署configmap
$ kubectl apply -f re-cg.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: bpae-rabbitmq-config
namespace: rq
labels:
appname: rabbitmq
app: bpae-rabbitmq-config
data:
enabled_plugins: |
[rabbitmq_management,rabbitmq_peer_discovery_k8s].
rabbitmq.conf: |
cluster_name = bpae-rabbitmq
listeners.tcp.default = 5672
default_vhost = /
default_user = xx #用户名
default_pass = xx #密码
default_user_tags.administrator = true
default_user_tags.management = true
default_user_tags.custom_tag = true
channel_max = 1024
tcp_listen_options.backlog = 2048
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
cluster_formation.k8s.host = kubernetes.default.svc.cluster.local
cluster_formation.k8s.address_type = hostname
cluster_formation.node_cleanup.interval = 30
cluster_formation.node_cleanup.only_log_warning = true
cluster_partition_handling = autoheal
queue_master_locator=min-masters
loopback_users.guest = false
cluster_formation.k8s.hostname_suffix = .autozx-rabbitmq.rq.svc.cluster.local
部署 Headless Service（提供集群 DNS)
$ kubectl apply -f bpae-rabbitmq-headless.yaml
apiVersion: v1
kind: Service
metadata:
name: autozx-rabbitmq # 与 StatefulSet 的 K8S_SERVICE_NAME 一致
namespace: rq
labels:
appname: rabbitmq
app: bpae-rabbitmq
spec:
selector:
app: bpae-rabbitmq # 匹配 StatefulSet 的 Pod 标签
clusterIP: None # 无头服务核心标识
ports:
- name: amqp
port: 5672
targetPort: amqp
- name: http
port: 15672
targetPort: http
- name: epmd
port: 4369
targetPort: epmd # 集群发现端口
- name: cluster-comm
port: 25672
targetPort: cluster-comm # 集群内部通信端口
部署 StatefulSet + NodePort Service（启动 RabbitMQ 集群）
$ kubectl apply -f sta-svc.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: bpae-rabbitmq
namespace: rq
labels:
appname: rabbitmq
app: bpae-rabbitmq
spec:
serviceName: "autozx-rabbitmq"
replicas: 3
selector:
matchLabels:
app: bpae-rabbitmq
template:
metadata:
labels:
app: bpae-rabbitmq
spec:
securityContext:
runAsUser: 0
runAsGroup: 0
fsGroup: 0
containers:
- name: rabbitmq-server
image: images.bz-link.net/images/bpae-rabbitmq:v1.0 # 你的镜像
imagePullPolicy: IfNotPresent
env:
- name: K8S_SERVICE_NAME
value: autozx-rabbitmq
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: RABBITMQ_USE_LONGNAME
value: "true"
- name: RABBITMQ_NODENAME
value: rabbit@$(POD_NAME).$(K8S_SERVICE_NAME).$(POD_NAMESPACE).svc.cluster.local
- name: ERLANG_COOKIE_VALUE
value: "YZSDHWMFSMKEMBDHSGGZ"
ports:
- name: http
containerPort: 15672
- name: amqp
containerPort: 5672
- name: epmd
containerPort: 4369
- name: cluster-comm
containerPort: 25672
command: ["/bin/sh", "-c"]
args:
- |
# 1. 生成并验证 Cookie 文件（权限已正常）
rm -rf /var/lib/rabbitmq/.erlang.cookie
echo "$ERLANG_COOKIE_VALUE" > /var/lib/rabbitmq/.erlang.cookie
chmod 600 /var/lib/rabbitmq/.erlang.cookie
chown 999:999 /var/lib/rabbitmq/.erlang.cookie
# 2. 自动查找 rabbitmq-server 实际路径（关键步骤）
echo "=== 正在查找 rabbitmq-server 实际路径 ==="
RABBITMQ_SERVER_PATH=$(find / -name rabbitmq-server 2>/dev/null | head -n 1)
echo "找到的路径：$RABBITMQ_SERVER_PATH"
# 3. 验证路径是否存在
if [ -z "$RABBITMQ_SERVER_PATH" ] || [ ! -x "$RABBITMQ_SERVER_PATH" ]; then
echo "ERROR：未找到可执行的 rabbitmq-server 命令！"
exit 1
fi
echo "=== 路径验证通过，开始启动服务 ==="
# 4. 用找到的实际路径启动服务
$RABBITMQ_SERVER_PATH --user rabbitmq
volumeMounts:
- name: rbmq-data
mountPath: /var/lib/rabbitmq
- name: rabbitmq-config-volume
mountPath: /etc/rabbitmq/
resources:
requests:
cpu: "1.5"
memory: "3Gi"
limits:
cpu: "2"
memory: "4Gi"
securityContext:
allowPrivilegeEscalation: true
privileged: false
restartPolicy: Always
serviceAccountName: rabbitmq-cluster
terminationGracePeriodSeconds: 60
volumes:
- name: rabbitmq-config-volume
configMap:
name: bpae-rabbitmq-config
volumeClaimTemplates:
- metadata:
name: rbmq-data
spec:
accessModes:
- ReadWriteOnce
storageClassName: nfs-client
resources:
requests:
storage: 20Gi
---
# NodePort Service 不变
apiVersion: v1
kind: Service
metadata:
name: bpae-rabbitmq-nodeport
namespace: rq
labels:
appname: rabbitmq
app: bpae-rabbitmq
spec:
ports:
- name: amqp
port: 5672
targetPort: amqp
nodePort: 30567
- name: http
port: 15672
targetPort: http
nodePort: 31672
- name: epmd
port: 4369
targetPort: epmd
- name: cluster-comm
port: 25672
targetPort: cluster-comm
selector:
app: bpae-rabbitmq
type: NodePort
检查 Pod 状态（等待 3 个 Pod 均为Running/1/1)
kubectl get pods -n rq -l app=bpae-rabbitmq -w
验证集群状态（确认 3 个节点组成集群）
kubectl exec -it bpae-rabbitmq-0 -n rq -- /opt/rabbitmq/sbin/rabbitmqctl cluster_status
配置高可用队列（避免单点故障）
# 在主节点执行（配置所有队列自动镜像到所有节点）
kubectl exec -it bpae-rabbitmq-0 -n rq -- /opt/rabbitmq/sbin/rabbitmqctl set_policy ha-all "^" '{"ha-mode":"all", "ha-sync-mode":"automatic"}'
验证管理界面
http://IP:15672 账号/密码：xx/xx
更多推荐
4
0- 0
已为社区贡献6条内容
所有评论(0)