StatefulSet应用案例-Redis一主两从三哨兵

299 views

一、部署思路

  1. 一共6个容器，1个主节点、2个从节点、3个哨兵节点。
  2. 编写namespace脚本，创建专门的namespace
  3. 编写configmap脚本，分别将主节点、从节点、哨兵的配置写在里面
  4. 编写secret脚本，将redis的密码保存在里面
  5. 编写pv脚本，用于创建磁盘资源
  6. 编写service脚本，1个提供redis之间的网络，1个提供哨兵之间的网络，1个对外提供服务
  7. 编写statefulSet脚本，1个提供redis主从服务，1个提供哨兵集群服务

二、部署

部署说明

软件名称 软件版本
redis v6.2.5
kubernetes v20.10.17
docker v1.23.10

部署条件

  1. 有个k8s集群
  2. k8s集群集成了nfs之类的作为存储抽象

1、编写namespace脚本

01-redis-namespace.yaml

apiVersion: v1
#创建Namespace类型资源
kind: Namespace
metadata:
  #资源名称
  name: redis
  labels:
    app: redis
# kubectl create ns redis

相关命令

#执行命令
kubectl apply -f 01-redis-namespace.yaml
#查看命名空间
kubectl get ns

2、编写configmap脚本

02-redis-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
data:
  #这里定义了多个数据信息
  master.conf: |
    # Master配置
    requirepass redisPassword
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    replica-read-only no
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  slave.conf: |
    # Slave配置
    replicaof redis-0.redis-svc 6379
    masterauth redisPassword
    requirepass redisPassword
    replica-read-only yes
    bind 0.0.0.0
    daemonize no
    protected-mode yes
    port 6379
    tcp-backlog 511
    timeout 0
    tcp-keepalive 300
    pidfile /var/run/redis_6379.pid
    loglevel notice
    logfile ""
    databases 16
    always-show-logo no
    set-proc-title yes
    proc-title-template "{title} {listen-addr} {server-mode}"
    stop-writes-on-bgsave-error yes
    rdbcompression yes
    rdbchecksum yes
    dbfilename dump.rdb
    rdb-del-sync-files no
    dir /data
    replica-serve-stale-data yes
    repl-diskless-sync no
    repl-diskless-sync-delay 5
    repl-diskless-load disabled
    repl-disable-tcp-nodelay no
    replica-priority 100
    acllog-max-len 128
    lazyfree-lazy-eviction no
    lazyfree-lazy-expire no
    lazyfree-lazy-server-del no
    replica-lazy-flush no
    lazyfree-lazy-user-del no
    lazyfree-lazy-user-flush no
    oom-score-adj no
    oom-score-adj-values 0 200 800
    disable-thp yes
    appendonly no
    appendfsync everysec
    no-appendfsync-on-rewrite no
    auto-aof-rewrite-percentage 100
    auto-aof-rewrite-min-size 64mb
    aof-load-truncated yes
    aof-use-rdb-preamble yes
    lua-time-limit 5000
    slowlog-log-slower-than 10000
    slowlog-max-len 128
    latency-monitor-threshold 0
    notify-keyspace-events ""
    hash-max-ziplist-entries 512
    hash-max-ziplist-value 64
    list-max-ziplist-size -2
    list-compress-depth 0
    set-max-intset-entries 512
    zset-max-ziplist-entries 128
    zset-max-ziplist-value 64
    hll-sparse-max-bytes 3000
    stream-node-max-bytes 4096
    stream-node-max-entries 100
    activerehashing yes
    client-output-buffer-limit normal 0 0 0
    client-output-buffer-limit replica 256mb 64mb 60
    client-output-buffer-limit pubsub 32mb 8mb 60
    hz 10
    dynamic-hz yes
    aof-rewrite-incremental-fsync yes
    rdb-save-incremental-fsync yes
    jemalloc-bg-thread yes
  sentinel.conf: |
    # 哨兵配置
    port 26379
    daemonize no
    pidfile "/var/run/redis-sentinel.pid"
    logfile ""
    dir "/data"
    sentinel monitor master0 redis-0.redis-svc 6379 2
    sentinel auth-pass master0 a123456!
    sentinel down-after-milliseconds master0 30000
    sentinel parallel-syncs master0 1
    sentinel failover-timeout master0 180000
    acllog-max-len 128
    sentinel deny-scripts-reconfig yes
    sentinel resolve-hostnames yes
    sentinel announce-hostnames no
    protected-mode no
    user default on nopass sanitize-payload ~* &* +@all

相关命令

#执行命令
kubectl apply -f 02-redis-configmap.yaml
#查看redis命名空间下的configmap
kubectl get cm -n redis
#查看redis命名空间下名为redis的configmap详情
kubectl describe configmap redis -n redis

3、编写secret脚本

03-redis-secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  namespace: redis
  labels:
    app: redis
#Opaque 类型的数据是一个 map 类型，要求value是base64编码。
type: Opaque
data:
  redisPassword: YTEyMzQ1NiE= #123456转成base64 echo -n "a123456!" | base64

相关命令

#执行命令
kubectl apply -f 03-redis-secret.yaml
#查看redis命名空间下的configmap
kubectl get secret -n redis
#查看redis命名空间下名为redis-secret的secret详情
kubectl describe secret redis-secret -n redis

4、编写PV脚本

采用NFS方式作为网络存储，后续会自动生成pvc

所有的k8s节点上都要安装nfs，NFS搭建略过，信息如下

IP: 192.168.8.201
mkdir -p /data/redis/nfs-{1..12}

vim /etc/exports 
/data/redis/nfs-1  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-2  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-3  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-4  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-5  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-6  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-7  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-8  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-9  *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-10 *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-11 *(rw,no_root_squash,no_all_squash,sync)
/data/redis/nfs-12 *(rw,no_root_squash,no_all_squash,sync)

systemctl restart nfs-server.service

1）编写ServiceAccount、ClusterRole、ClusterRoleBinding、Role、RoleBinding脚本管理NFS

04-redis-pv.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv001
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-1
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv002
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-2
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv003
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-3
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv004
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path:  /data/redis/nfs-4
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv005
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-5
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv006
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-6
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv007
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-7
    server: 192.168.8.201

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv008
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-8
    server: 192.168.8.201

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv009
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-9
    server: 192.168.8.201
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv010
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-10
    server: 192.168.8.201

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv011
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-11
    server: 192.168.8.201

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-pv012
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: managed-nfs-storage
  nfs:
    path: /data/redis/nfs-12
    server: 192.168.8.201

相关命令

#执行命令
kubectl apply -f 04-redis-pv.yaml

5、编写Service脚本

05-redis-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: redis-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    #匹配带有app: redis标签的pod
    app: redis
  clusterIP: None
  ports:
  - name: redis
    port: 6379
---
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    #匹配带有app: redis标签的pod
    app: redis-sentinel
  clusterIP: None
  ports:
  - name: redis
    port: 26379
---
apiVersion: v1
kind: Service
metadata:
  name: redis-sentinel-svc-nodeport
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    app: redis-sentinel
  type: NodePort
  ports:
  #集群内访问  <集群内部ip>:26379  会被代理到对应pod的26379端口
  #集群外访问  <k8s的ip>:30379   会被代理到对应pod的26379端口
  - name: redis-sentinel
    #集群内部之间访问使用的端口
    port: 26379
    #pod上需要被代理的端口
    targetPort: 26379
    #集群外访问使用的端口，范围固定 30000 ~ 32767
    nodePort: 30379

相关命令

#执行命令
kubectl apply -f 05-redis-service.yaml
#查看redis命名空间下service信息
kubectl get svc -n redis

6、编写redis的StatefulSet脚本

06-redis-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
  namespace: redis
  labels:
    app: redis
spec:
  selector:
    matchLabels:
      app: redis
  #与redis-service.yaml中的保持一致
  serviceName: redis-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis
    spec:
      initContainers:
      - name: init-redis
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          #从pod的hostname中通过正则获取序号，如果没有截取到就退出程序
          ordinal=`hostname | awk -F"-" '{print $2}'`
          if [ ${ordinal} -eq 0 ]; then
            # 如果Pod的序号为0，说明它是Master节点
            cp /mnt/config-map/master.conf /conf/redis.conf
          else
            # 否则，拷贝ConfigMap里的Slave的配置文件
            cp /mnt/config-map/slave.conf /conf/redis.conf
          fi
          sed -i "s/redisPassword/${REDIS_PASSWORD}/g"  /conf/redis.conf
        env:
        - name: REDIS_PASSWORD
          valueFrom:
            secretKeyRef:
              name: redis-secret
              key: redisPassword
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis
        image: redis:6.2.5
        command: ["redis-server"]
        args:
          - "/conf/redis.conf"
        ports:
        - name: redis
          containerPort: 6379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 100Mi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 5Gi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi

相关命令

#执行命令
kubectl apply -f 06-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis

7、编写哨兵的StatefulSet脚本

07-redis-sentinel-statefulset.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-sentinel
  namespace: redis
  labels:
    app: redis-sentinel
spec:
  selector:
    matchLabels:
      app: redis-sentinel
  #与redis-service.yaml中的保持一致
  serviceName: redis-sentinel-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: redis-sentinel
    spec:
      initContainers:
      - name: init-redis-sentinel
        image: redis:6.2.5
        command: 
        - bash
        - "-c"
        - |
          set -ex
          cp /mnt/config-map/sentinel.conf /conf/redis-sentinel.conf
        volumeMounts:
        - name: conf
          mountPath: /conf
        - name: config-map
          mountPath: /mnt/config-map
      containers:
      - name: redis-sentinel
        image: redis:6.2.5
        command: ["redis-sentinel"]
        args:
          - "/conf/redis-sentinel.conf"
        ports:
        - name: redis-sentinel
          containerPort: 26379
        volumeMounts:
        - name: data
          mountPath: /data
        - name: conf
          mountPath: /conf
        resources:
          requests:
            cpu: 500m
            memory: 100Mi
      volumes:
      - name: config-map
        #这个卷挂载到configMap上
        configMap:
          name: redis
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 100Mi
  - metadata: 
      name: conf
    spec:
      accessModes:
      - ReadWriteOnce
      #与nfs-StorageClass.yaml metadata.name保持一致
      storageClassName: managed-nfs-storage
      resources:
        requests:
          storage: 10Mi

相关命令

#执行命令
kubectl apply -f 07-redis-statefulset.yaml
#查看redis命名空间下pvc信息
kubectl get pvc -n redis
kubectl describe pvc data-redis-0  -n redis
#查看redis命名空间下pv信息
kubectl get pv -n redis
#查看redis命名空间下pod节点信息
kubectl get pod -n redis