基于k8s集群的redis-cluster集群
一、提前准备好nfs存储
[root@k8s-master ~]# yum -y install nfs-utils nfs-common rpcbind
[root@k8s-master ~]# mkdir /nfsdata
[root@k8s-master ~]# chmod -R 777 /nfsdata
[root@k8s-master ~]# vim /etc/exports
/nfsdata *(rw,no_root_squash,sync,no_all_squash)
[root@k8s-master ~]# systemctl start rpcbind nfs
[root@k8s-master ~]# systemctl status nfs# node节点
yum -y install nfs-utils nfs-common二、制作动态存储
1.安装helm工具
[root@k8s-master ~]# wget https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
--2024-08-06 15:53:55-- https://get.helm.sh/helm-v3.10.1-linux-amd64.tar.gz
正在解析主机 get.helm.sh (get.helm.sh)... 152.199.39.108, 2606:2800:247:1cb7:261b:1f9c:2074:3c
正在连接 get.helm.sh (get.helm.sh)|152.199.39.108|:443... 已连接。
已发出 HTTP 请求，正在等待回应... 200 OK
长度：14565908 (14M) [application/x-tar]
正在保存至: “helm-v3.10.1-linux-amd64.tar.gz”100%[=======================================================================>] 14,565,908 157KB/s 用时 46s 2024-08-06 15:54:43 (312 KB/s) - 已保存 “helm-v3.10.1-linux-amd64.tar.gz” [14565908/14565908])[root@k8s-master ~]# tar -xzvf helm-v3.10.1-linux-amd64.tar.gz
linux-amd64/
linux-amd64/helm
linux-amd64/LICENSE
linux-amd64/README.md
[root@k8s-master ~]# cp linux-amd64/helm /usr/bin/
[root@k8s-master ~]# helm version
version.BuildInfo{Version:"v3.10.1", GitCommit:"9f88ccb6aee40b9a0535fcc7efea6055e1ef72c9", GitTreeState:"clean", GoVersion:"go1.18.7"}
2.配置helm下载源
[root@k8s-master ~]# helm repo add stable http://mirror.azure.cn/kubernetes/charts
"stable" has been added to your repositories
[root@k8s-master ~]# helm repo list
NAME URL
stable http://mirror.azure.cn/kubernetes/charts
3.创建storageclass与nfs的目录相关联
[root@k8s-master ~]# helm install nfs-redis stable/nfs-client-provisioner --set nfs.server=192.168.22.134 --set nfs.path=/nfsdata
WARNING: This chart is deprecated
NAME: nfs-redis
LAST DEPLOYED: Tue Aug 6 16:01:14 2024
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
[root@k8s-master ~]# helm list
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
nfs-redis default 1 2024-08-06 16:01:14.424540014 +0800 CST deployed nfs-client-provisioner-1.2.11 3.1.0
[root@k8s-master ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks 1/1 Running 0 39s[root@k8s-master ~]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-client cluster.local/nfs-redis-nfs-client-provisioner Delete Immediate true 53s
三、redis配置文件configmap
[root@k8s-master redis]# vim redis.conf
[root@k8s-master redis]# cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379
[root@k8s-master redis]# kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created
[root@k8s-master redis]# kubectl get cf
error: the server doesn't have a resource type "cf"
[root@k8s-master redis]# kubectl get cm
NAME DATA AGE
redis-conf 1 19s
四、创建Headless service
[root@k8s-master redis]# vim headless-service.yaml
[root@k8s-master redis]# cat headless-service.yaml
apiVersion: v1
kind: Service
metadata:name: redis-servicelabels:app: redis
spec:ports:- name: redis-portport: 6379clusterIP: Noneselector:app: redisappCluster: redis-cluster
[root@k8s-master redis]# kubectl apply -f headless-service.yaml
service/redis-service created
[root@k8s-master redis]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 40h
redis-service ClusterIP None <none> 6379/TCP 9s
五、statefulSet运行redis实例
[root@k8s-master redis]# vim statefulset-redis.yaml
[root@k8s-master redis]# cat statefulset-redis.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:name: redis-app
spec:serviceName: "redis-service"replicas: 6selector:matchLabels:app: redisappCluster: redis-clustertemplate:metadata:labels:app: redisappCluster: redis-clusterspec:terminationGracePeriodSeconds: 20affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight: 100podAffinityTerm:labelSelector:matchExpressions:- key: appoperator: Invalues:- redistopologyKey: kubernetes.io/hostnamecontainers:- name: redisimage: daocloud.io/library/redis:6-alpine3.12command:- "redis-server"args:- "/etc/redis/redis.conf"- "--protected-mode"- "no"resources:requests:cpu: "100m"memory: "100Mi"ports:- name: rediscontainerPort: 6379protocol: "TCP"- name: clustercontainerPort: 16379protocol: "TCP"volumeMounts:- name: "redis-conf"mountPath: "/etc/redis"- name: "redis-data"mountPath: "/var/lib/redis"volumes:- name: "redis-conf"configMap:name: "redis-conf"items:- key: "redis.conf"path: "redis.conf"volumeClaimTemplates:- metadata:name: redis-dataspec:accessModes: [ "ReadWriteMany" ]storageClassName: "nfs-client"resources:requests:storage: 200M
[root@k8s-master redis]# kubectl apply -f statefulset-redis.yaml
statefulset.apps/redis-app created
[root@k8s-master redis]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks 1/1 Running 0 31m
redis-app-0 1/1 Running 0 29s
redis-app-1 1/1 Running 0 26s
redis-app-2 1/1 Running 0 21s
redis-app-3 1/1 Running 0 17s
redis-app-4 1/1 Running 0 12s
redis-app-5 1/1 Running 0 8s
[root@k8s-master redis]# kubectl get statefulset
NAME READY AGE
redis-app 6/6 68s
六、验证唯一访问标识可用性
每个Pod都会得到集群内的⼀个DNS域名，格式为
(podname).(servicename).(namespace).svc.cluster.local
例如： redis-app-0.redis-service.default.svc.cluster.local
redis-app-1.redis-service.default.svc.cluster.local
# 创建一个pod用于测试，测试完即可删除
[root@k8s-master redis]# vim busybox.yaml
[root@k8s-master redis]# cat busybox.yaml
---
apiVersion: v1
kind: Pod
metadata:name: busybox
spec:containers:- name: busyboximage: daocloud.io/library/busyboxstdin: truetty: true
[root@k8s-master redis]# kubectl apply -f busybox.yaml
pod/busybox created
[root@k8s-master redis]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
busybox 1/1 Running 0 2m56s 10.244.1.9 k8s-node1 <none> <none>
nfs-redis-nfs-client-provisioner-7bd85b55f4-p26ks 1/1 Running 0 51m 10.244.2.4 k8s-node2 <none> <none>
redis-app-0 1/1 Running 0 20m 10.244.1.4 k8s-node1 <none> <none>
redis-app-1 1/1 Running 0 20m 10.244.2.5 k8s-node2 <none> <none>
redis-app-2 1/1 Running 0 20m 10.244.1.5 k8s-node1 <none> <none>
redis-app-3 1/1 Running 0 19m 10.244.2.6 k8s-node2 <none> <none>
redis-app-4 1/1 Running 0 19m 10.244.1.6 k8s-node1 <none> <none>
redis-app-5 1/1 Running 0 19m 10.244.2.7 k8s-node2 <none> <none>[root@k8s-master redis]# kubectl exec -it busybox /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ping redis-app-0.redis-service.default.svc.cluster.local
PING redis-app-0.redis-service.default.svc.cluster.local (10.244.1.4): 56 data bytes
64 bytes from 10.244.1.4: seq=0 ttl=64 time=0.132 ms
64 bytes from 10.244.1.4: seq=1 ttl=64 time=0.062 ms
64 bytes from 10.244.1.4: seq=2 ttl=64 time=0.074 ms
^C
--- redis-app-0.redis-service.default.svc.cluster.local ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.062/0.089/0.132 ms
/ # ping redis-app-1.redis-service.default.svc.cluster.local
PING redis-app-1.redis-service.default.svc.cluster.local (10.244.2.5): 56 data bytes
64 bytes from 10.244.2.5: seq=0 ttl=62 time=2.076 ms
64 bytes from 10.244.2.5: seq=1 ttl=62 time=0.796 ms
^C
--- redis-app-1.redis-service.default.svc.cluster.local ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 0.796/1.456/2.108 ms
/ # ping redis-app-2.redis-service.default.svc.cluster.local
PING redis-app-2.redis-service.default.svc.cluster.local (10.244.1.5): 56 data bytes
64 bytes from 10.244.1.5: seq=0 ttl=64 time=0.167 ms
64 bytes from 10.244.1.5: seq=1 ttl=64 time=0.067 ms
64 bytes from 10.244.1.5: seq=2 ttl=64 time=0.061 ms
^C
--- redis-app-2.redis-service.default.svc.cluster.local ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max = 0.061/0.098/0.167 ms
/ #
查看PV和PVC
[root@k8s-master redis]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-04c6e1bc-709f-4a77-bd36-7cd8c23d4ca1 200M RWX Delete Bound default/redis-data-redis-app-1 nfs-client 21m
pvc-1650521c-ba23-4502-b8e8-c260359a2d27 200M RWX Delete Bound default/redis-data-redis-app-0 nfs-client 21m
pvc-63d54f9b-7be4-4ce7-a72b-caa7bedde507 200M RWX Delete Bound default/redis-data-redis-app-2 nfs-client 21m
pvc-652e836c-f426-4629-8600-d542f4269a52 200M RWX Delete Bound default/redis-data-redis-app-3 nfs-client 21m
pvc-b1eee630-b2fc-4423-be6c-32278d6a8018 200M RWX Delete Bound default/redis-data-redis-app-5 nfs-client 21m
pvc-efc54629-d01d-45ac-9c42-37ecbe34189a 200M RWX Delete Bound default/redis-data-redis-app-4 nfs-client 21m
[root@k8s-master redis]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
redis-data-redis-app-0 Bound pvc-1650521c-ba23-4502-b8e8-c260359a2d27 200M RWX nfs-client 21m
redis-data-redis-app-1 Bound pvc-04c6e1bc-709f-4a77-bd36-7cd8c23d4ca1 200M RWX nfs-client 21m
redis-data-redis-app-2 Bound pvc-63d54f9b-7be4-4ce7-a72b-caa7bedde507 200M RWX nfs-client 21m
redis-data-redis-app-3 Bound pvc-652e836c-f426-4629-8600-d542f4269a52 200M RWX nfs-client 21m
redis-data-redis-app-4 Bound pvc-efc54629-d01d-45ac-9c42-37ecbe34189a 200M RWX nfs-client 21m
redis-data-redis-app-5 Bound pvc-b1eee630-b2fc-4423-be6c-32278d6a8018 200M RWX nfs-client 21m
七、redis集群初始化
由于Redis集群必须在所有节点启动后才能进⾏初始化，这⾥，我们专⻔启动⼀个Ubuntu的容器，可以在该容器中安装Redis-tribe，进⽽初始化Redis集群；
1）创建Ubuntu容器并配置源
[root@k8s-master redis]# kubectl run -it ubuntu --image=daocloud.io/library/ubuntu:artful-20170619 --restart=Never /bin/bash
If you don't see a command prompt, try pressing enter.
root@ubuntu:/#
# 添加阿里云的Ubuntu源，并安装基本的软件环境
root@ubuntu:/# root@ubuntu:/# cat > /etc/apt/sources.list << EOF
> deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
> deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
> deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
> EOF
root@ubuntu:/# apt-get update......
root@ubuntu:/# echo $?
0
root@ubuntu:/# apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils2）安装redis-trib
root@ubuntu:/# pip install redis-trib==0.5.1
3）创建只有master节点的集群
root@ubuntu:/# redis-trib.py create \
`dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
`dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
`dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379
4）为每个master添加slave
root@ubuntu:/# redis-trib.py replicate \
--master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379
root@ubuntu:/# redis-trib.py replicate \
--master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379
root@ubuntu:/# redis-trib.py replicate \
--master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
--slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379
5） 连接到任意一个redis pod中检验一下
[root@k8s-master redis]# kubectl exec -it redis-app-5 /bin/sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/data # /usr/local/bin/redis-cli -c
127.0.0.1:6379> CLUSTER NODES
42beeed3837a809034874d7415c307e91e2daac8 10.244.2.5:6379@16379 master - 0 1722937557103 1 connected 0-5461
c06e4e94182f72196bcf524e948c6265a8fcfb87 10.244.1.5:6379@16379 master - 0 1722937556000 4 connected 10923-16383
9bcbf04efa0705a10ab9c80b5379f346f2e32c38 10.244.2.7:6379@16379 myself,slave c06e4e94182f72196bcf524e948c6265a8fcfb87 0 1722937555000 4 connected
79ebf7ed87288fe7b23065c5e8911878b8dde0e4 10.244.2.6:6379@16379 slave 3413f32da2cb4e177b2a6c57ffd1171850c039c2 0 1722937556296 2 connected
f62ab0f4d9e3ac82163d63310716cc460ac31898 10.244.1.6:6379@16379 slave 42beeed3837a809034874d7415c307e91e2daac8 0 1722937555792 1 connected
3413f32da2cb4e177b2a6c57ffd1171850c039c2 10.244.1.4:6379@16379 master - 0 1722937556802 2 connected 5462-10922
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:4
cluster_my_epoch:4
cluster_stats_messages_ping_sent:654
cluster_stats_messages_pong_sent:582
cluster_stats_messages_sent:1236
cluster_stats_messages_ping_received:581
cluster_stats_messages_pong_received:654
cluster_stats_messages_meet_received:1
cluster_stats_messages_received:1236
127.0.0.1:6379>
八、创建用于访问Service
还需要创建⼀个Service，专⽤于为Redis集群提供访问和负载均衡；
[root@k8s-master redis]# vim redis-access-service.yaml
[root@k8s-master redis]# cat redis-access-service.yaml
apiVersion: v1
kind: Service
metadata:name: redis-access-servicelabels:app: redis
spec:ports:- name: redis-portprotocol: "TCP"port: 6379targetPort: 6379selector:app: redisappCluster: redis-cluster
[root@k8s-master redis]# kubectl apply -f redis-access-service.yaml
service/redis-access-service created
[root@k8s-master redis]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 42h
redis-access-service ClusterIP 10.103.21.164 <none> 6379/TCP 11s
redis-service ClusterIP None <none> 6379/TCP 103m
[root@k8s-master redis]# kubectl get ep
NAME ENDPOINTS AGE
cluster.local-nfs-redis-nfs-client-provisioner <none> 116m
kubernetes 192.168.22.134:6443 42h
redis-access-service 10.244.1.4:6379,10.244.1.5:6379,10.244.1.6:6379 + 3 more... 25s
redis-service 10.244.1.4:6379,10.244.1.5:6379,10.244.1.6:6379 + 3 more... 103m