完整的脚本下载,前往gitee:https://gitee.com/qiaodaimadewangcai/study-notes/tree/master/k8s/k8s部署redis主从+哨兵/yaml

一、部署思路

  1. 一共6个容器,1个主节点、2个从节点、3个哨兵节点。
  2. 编写namespace脚本,创建专门的namespace
  3. 编写configmap脚本,分别将主节点、从节点、哨兵的配置写在里面
  4. 编写secret脚本,将redis的密码保存在里面
  5. 编写storageClass脚本,用于自动创建磁盘资源
  6. 编写service脚本,1个提供redis之间的网络,1个提供哨兵之间的网络,1个对外提供服务
  7. 编写statefulSet脚本,1个提供redis主从服务,1个提供哨兵集群服务

二、部署

部署说明

软件名称软件版本
redisv6.2.5
kubernetesv20.10.17
dockerv1.23.10

部署条件

  1. 有个k8s集群
  2. k8s集群集成了nfs之类的作为存储抽象

1、编写namespace脚本

01-redis-namespace.yaml

apiVersion: v1
#创建Namespace类型资源
kind: Namespace
metadata:
  #资源名称
  name: redis
  labels:
    app: redis

相关命令

#执行命令
kubectl apply -f 01-redis-namespace.yaml
#查看命名空间
kubectl get ns

2、编写configmap脚本

02-redis-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
data:
  #这里定义了多个数据信息
  master.conf: |
    # Master配置
    requirepass redisPassword
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    replica-read-only no
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  slave.conf: |
    # Slave配置
    replicaof redis-0.redis-svc 6379
    masterauth redisPassword
    requirepass redisPassword
    replica-read-only yes
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  sentinel.conf: |
    # 哨兵配置
    port 26379
    daemonize no
    pidfile "/var/run/redis-sentinel.pid"
    logfile ""
    dir "/data"
    sentinel monitor master0 redis-0.redis-svc 6379 2
    sentinel auth-pass master0 a123456!
    sentinel down-after-milliseconds master0 30000
    sentinel parallel-syncs master0 1
    sentinel failover-timeout master0 180000
    acllog-max-len 128
    sentinel deny-scripts-reconfig yes
    sentinel resolve-hostnames yes
    sentinel announce-hostnames no
    protected-mode no
    user default on nopass sanitize-payload ~* &* +@all

相关命令

#执行命令
kubectl apply -f 02-redis-configmap.yaml
#查看redis命名空间下的configmap
kubectl get cm -n redis
#查看redis命名空间下名为redis的configmap详情
kubectl describe configmap redis -n redis

3、编写secret脚本

03-redis-secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  namespace: redis
  labels:
    app: redis
#Opaque 类型的数据是一个 map 类型,要求value是base64编码。
type: Opaque
data:
  redisPassword: YTEyMzQ1NiE= #123456转成base64 echo -n "a123456!" | base64

相关命令

#执行命令
kubectl apply -f 03-redis-secret.yaml
#查看redis命名空间下的configmap
kubectl get secret -n redis
#查看redis命名空间下名为redis-secret的secret详情
kubectl describe secret redis-secret -n redis

4、编写StorageClass脚本

采用StorageClass+NFS方式作为网络存储,后续会自动生成pvc和pv

所有的k8s节点上都要安装nfs,NFS搭建略过,信息如下

IP: 192.168.56.80
Export PATH: /mnt

1)编写ServiceAccount、ClusterRole、ClusterRoleBinding、Role、RoleBinding脚本管理NFS

04-redis-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  namespace: redis
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: redis
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: redis
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: redis
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

相关命令

#执行命令
kubectl apply -f 04-redis-rbac.yaml

2)编写StorageClass脚本

05-redis-nfs-storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: redis-nfs-storage #这里的名称要和provisioner配置文件中的环境变量PROVISIONER_NAME保持一致
volumeBindingMode: WaitForFirstConsumer
parameters:
  archiveOnDelete: "true" #pvc被删除了也需要进行存档

相关命令

#执行命令
kubectl apply -f 05-redis-nfs-storageclass.yaml
#查看StorageClass信息
kubectl get sc

3)编写nfs-provisioner的Deployment脚本

所有的k8s节点上都要安装nfs,不然这段就会出问题无法运行

06-redis-nfs-provisioner-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  namespace: redis  #与RBAC文件中的namespace保持一致
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nfs-client-provisioner
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: redis-nfs-storage  #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
            - name: NFS_SERVER
              value: 192.168.56.80   #NFS Server IP地址
            - name: NFS_PATH  
              value: /mnt    #NFS挂载卷
      volumes:
        - name: nfs-client-root
          nfs:
            server: 192.168.56.80  #NFS Server IP地址
            path: /mnt     #NFS 挂载卷

相关命令

#执行命令
kubectl apply -f 06-redis-nfs-provisioner-deployment.yaml

5、编写Service脚本

07-redis-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: redis-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
  	#匹配带有app: redis标签的pod
    app: redis
  clusterIP: None
  ports:
  - name: redis
    port: 6379
-----------------------------------------
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
  	#匹配带有app: redis标签的pod
    app: redis-sentinel
  clusterIP: None
  ports:
  - name: redis
    port: 26379
------------------------------------------
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc-nodeport
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    app: redis-sentinel
  type: NodePort
  ports:
  #集群内访问  <集群内部ip>:26379  会被代理到对应pod的26379端口
  #集群外访问  <k8s的ip>:30379   会被代理到对应pod的26379端口
  - name: redisSentinel
    #集群内部之间访问使用的端口
    port: 26379
    #pod上需要被代理的端口
    targetPort: 26379
    #集群外访问使用的端口,范围固定 30000 ~ 32767
    nodePort: 30379

相关命令

#执行命令
kubectl apply -f 07-redis-service.yaml
#查看redis命名空间下service信息
kubectl get svc -n redis

6、编写redis的StatefulSet脚本

08-redis-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    matchLabels:
      app: redis
  #与redis-service.yaml中的保持一致
  serviceName: redis-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis
    spec:
      initContainers:
      - name: init-redis
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          #从pod的hostname中通过正则获取序号,如果没有截取到就退出程序
          ordinal=`hostname | awk -F"-" '{print $2}'`
          if [ ${ordinal} -eq 0 ]; then
            # 如果Pod的序号为0,说明它是Master节点
            cp /mnt/config-map/master.conf /conf/redis.conf
          else
            # 否则,拷贝ConfigMap里的Slave的配置文件
            cp /mnt/config-map/slave.conf /conf/redis.conf
          fi
          sed -i "s/redisPassword/${REDIS_PASSWORD}/g"  /conf/redis.conf
        env:
        - name: REDIS_PASSWORD
          valueFrom:
            secretKeyRef:
              name: redis-secret
              key: redisPassword
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis
        image: redis:6.2.5
        command: ["redis-server"]
        args:
          - "/conf/redis.conf"
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 2Gi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 5Gi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi

相关命令

#执行命令
kubectl apply -f 08-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis
#查看redis主从信息(redis客户端中使用)
info replication

7、编写哨兵的StatefulSet脚本

09-redis-sentinel-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-sentinel
  namespace: redis
  labels:
    app: redis-sentinel
spec:
  selector:
    matchLabels:
      app: redis-sentinel
  #与redis-service.yaml中的保持一致
  serviceName: redis-sentinel-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis-sentinel
    spec:
      initContainers:
      - name: init-redis-sentinel
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          cp /mnt/config-map/sentinel.conf /conf/redis-sentinel.conf
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis-sentinel
        image: redis:6.2.5
        command: ["redis-sentinel"]
        args:
          - "/conf/redis-sentinel.conf"
        ports:
        - name: redis-sentinel
          containerPort: 26379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 2Gi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 10Mi

相关命令

#执行命令
kubectl apply -f 09-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis
#查看redis哨兵信息(redis客户端中使用)
sentinel sentinels master0
Logo

K8S/Kubernetes社区为您提供最前沿的新闻资讯和知识内容

更多推荐