一、配置清单
1.基于K8s搭建部署1主2从3哨兵的Redis集群哨兵模式(集群内访问)
2.持久化数据选择用storageclass，动态创建pv存储，动态存储使用longhorn
创建redis配置文件
#注意内容中注释
apiVersion: v1
kind: Namespace
metadata:
name: prod
---
apiVersion: v1
kind: ConfigMap
metadata:
name: redis5-configmap
namespace: prod
labels:
app: redis
data:
redis.conf: |
dir "/data"
maxmemory 0
maxmemory-policy volatile-lru
min-slaves-max-lag 5
min-slaves-to-write 1
rdbchecksum yes
rdbcompression yes
repl-diskless-sync yes
save 900 1
sentinel.conf: |
dir "/data"
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 180000
sentinel parallel-syncs mymaster 5
init.sh: |
HOSTNAME="$(hostname)"
INDEX="${HOSTNAME##*-}"
#redis5-sentinel-svc和service名字相关
MASTER="$(redis-cli -h redis5-sentinel-svc -p 26379 sentinel get-master-addr-by-name mymaster | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"
MASTER_GROUP="mymaster"
QUORUM="2"
REDIS_CONF=/data/conf/redis.conf
REDIS_PORT=6379
SENTINEL_CONF=/data/conf/sentinel.conf
SENTINEL_PORT=26379
#需要修改为自己service名字无头服务的名字
SERVICE=redis5-sentinel-headless
set -eu
sentinel_update() {
echo "Updating sentinel config"
eval MY_SENTINEL_ID="\${SENTINEL_ID_$INDEX}"
sed -i "1s/^/sentinel myid $MY_SENTINEL_ID\\n/" "$SENTINEL_CONF"
sed -i "2s/^/sentinel monitor $MASTER_GROUP $1 $REDIS_PORT $QUORUM \\n/" "$SENTINEL_CONF"
echo "sentinel announce-ip $ANNOUNCE_IP" >> $SENTINEL_CONF
echo "sentinel announce-port $SENTINEL_PORT" >> $SENTINEL_CONF
}
redis_update() {
echo "Updating redis config"
echo "slaveof $1 $REDIS_PORT" >> "$REDIS_CONF"
echo "slave-announce-ip $ANNOUNCE_IP" >> $REDIS_CONF
echo "slave-announce-port $REDIS_PORT" >> $REDIS_CONF
}
copy_config() {
cp /readonly-config/redis.conf "$REDIS_CONF"
cp /readonly-config/sentinel.conf "$SENTINEL_CONF"
}
setup_defaults() {
echo "Setting up defaults"
if [ "$INDEX" = "0" ]; then
echo "Setting this pod as the default master"
redis_update "$ANNOUNCE_IP"
sentinel_update "$ANNOUNCE_IP"
sed -i "s/^.*slaveof.*//" "$REDIS_CONF"
else
# "redis-0.$SERVICE" 和statefulset的名字相关
DEFAULT_MASTER="$(getent hosts "redis5-sentinel-0.$SERVICE" | awk '{ print $1 }')"
if [ -z "$DEFAULT_MASTER" ]; then
echo "Unable to resolve host"
exit 1
fi
echo "Setting default slave config.."
redis_update "$DEFAULT_MASTER"
sentinel_update "$DEFAULT_MASTER"
fi
}
find_master() {
echo "Attempting to find master"
if [ "$(redis-cli -h "$MASTER" ping)" != "PONG" ]; then
echo "Can't ping master, attempting to force failover"
if redis-cli -h "$SERVICE" -p "$SENTINEL_PORT" sentinel failover "$MASTER_GROUP" | grep -q 'NOGOODSLAVE' ; then
setup_defaults
return 0
fi
sleep 10
MASTER="$(redis-cli -h $SERVICE -p $SENTINEL_PORT sentinel get-master-addr-by-name $MASTER_GROUP | grep -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"
if [ "$MASTER" ]; then
sentinel_update "$MASTER"
redis_update "$MASTER"
else
echo "Could not failover, exiting..."
exit 1
fi
else
echo "Found reachable master, updating config"
sentinel_update "$MASTER"
redis_update "$MASTER"
fi
}
mkdir -p /data/conf/
echo "Initializing config.."
copy_config
# "redis-$INDEX.$SERVICE" 和statefulset的名字相关
ANNOUNCE_IP=$(getent hosts "redis5-sentinel-$INDEX.$SERVICE" | awk '{ print $1 }')
if [ -z "$ANNOUNCE_IP" ]; then
"Could not resolve the announce ip for this pod"
exit 1
elif [ "$MASTER" ]; then
find_master
else
setup_defaults
fi
if [ "${AUTH:-}" ]; then
echo "Setting auth values"
ESCAPED_AUTH=$(echo "$AUTH" | sed -e 's/[\/&]/\\&/g');
sed -i "s/replace-default-auth/${ESCAPED_AUTH}/" "$REDIS_CONF" "$SENTINEL_CONF"
fi
echo "Ready..."
---
apiVersion: v1
kind: ConfigMap
metadata:
name: redis-probes
namespace: prod
labels:
app: redis
data:
check-quorum.sh: |
#!/bin/sh
set -eu
MASTER_GROUP="mymaster"
SENTINEL_PORT=26379
REDIS_PORT=6379
NUM_SLAVES=$(redis-cli -p "$SENTINEL_PORT" sentinel master mymaster | awk '/num-slaves/{getline; print}')
MIN_SLAVES=1
if [ "$1" = "$SENTINEL_PORT" ]; then
if redis-cli -p "$SENTINEL_PORT" sentinel ckquorum "$MASTER_GROUP" | grep -q NOQUORUM ; then
echo "ERROR: NOQUORUM. Sentinel quorum check failed, not enough sentinels found"
exit 1
fi
elif [ "$1" = "$REDIS_PORT" ]; then
if [ "$MIN_SLAVES" -gt "$NUM_SLAVES" ]; then
echo "Could not find enough replicating slaves. Needed $MIN_SLAVES but found $NUM_SLAVES"
exit 1
fi
fi
sh /probes/readiness.sh "$1"
readiness.sh: |
#!/bin/sh
set -eu
CHECK_SERVER="$(redis-cli -p "$1" ping)"
if [ "$CHECK_SERVER" != "PONG" ]; then
echo "Server check failed with: $CHECK_SERVER"
exit 1
fi
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: redis
namespace: prod
labels:
app: redis
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: redis
namespace: prod
labels:
app: redis
rules:
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: redis
namespace: prod
labels:
app: redis
subjects:
- kind: ServiceAccount
name: redis
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: redis
---
apiVersion: v1
kind: Service
metadata:
name: redis5-sentinel-headless
namespace: prod
labels:
app: redis-ha
annotations:
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
publishNotReadyAddresses: true
type: ClusterIP
clusterIP: None
ports:
- name: server
port: 6379
protocol: TCP
targetPort: redis
- name: sentinel
port: 26379
protocol: TCP
targetPort: sentinel
selector:
app: redis-ha
---
apiVersion: v1
kind: Service
metadata:
name: redis5-sentinel-svc
namespace: prod
labels:
app: redis-ha
annotations:
spec:
type: ClusterIP
ports:
- name: server
port: 6379
protocol: TCP
targetPort: redis
- name: sentinel
port: 26379
protocol: TCP
targetPort: sentinel
selector:
app: redis-ha
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis5-sentinel
namespace: prod
labels:
app: redis-ha
spec:
selector:
matchLabels:
app: redis-ha
serviceName: redis5-sentinel-headless
replicas: 3
podManagementPolicy: OrderedReady
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: redis-ha
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: redis-ha
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app: redis-ha
topologyKey: failure-domain.beta.kubernetes.io/zone
securityContext:
fsGroup: 1000
runAsNonRoot: true
runAsUser: 1000
serviceAccountName: redis
initContainers:
- name: config-init
image: redis:5.0.3
imagePullPolicy: IfNotPresent
resources:
{}
command:
- sh
args:
- /readonly-config/init.sh
env:
- name: SENTINEL_ID_0
value: 0c09a3866dba0f3b43ef2e383b5dc05980900fd8
- name: SENTINEL_ID_1
value: e6be0f70406122877338f7c814b17a7c7b648d82
- name: SENTINEL_ID_2
value: 31f8f52b34feaddcabdd6bf1827aeb02be44d2e3
volumeMounts:
- name: config
mountPath: /readonly-config
readOnly: true
- name: data
mountPath: /data
containers:
- name: redis
image: redis:5.0.3
imagePullPolicy: IfNotPresent
command:
- redis-server
args:
- /data/conf/redis.conf
livenessProbe:
exec:
command: [ "sh", "/probes/readiness.sh", "6379"]
initialDelaySeconds: 15
periodSeconds: 5
readinessProbe:
exec:
command: ["sh", "/probes/readiness.sh", "6379"]
initialDelaySeconds: 15
periodSeconds: 5
resources:
{}
ports:
- name: redis
containerPort: 6379
volumeMounts:
- mountPath: /data
name: data
- mountPath: /probes
name: probes
- name: sentinel
image: redis:5.0.3
imagePullPolicy: IfNotPresent
command:
- redis-sentinel
args:
- /data/conf/sentinel.conf
livenessProbe:
exec:
command: [ "sh", "/probes/readiness.sh", "26379"]
initialDelaySeconds: 15
periodSeconds: 5
readinessProbe:
exec:
command: ["sh", "/probes/readiness.sh", "26379"]
initialDelaySeconds: 15
periodSeconds: 5
resources:
{}
ports:
- name: sentinel
containerPort: 26379
volumeMounts:
- mountPath: /data
name: data
- mountPath: /probes
name: probes
volumes:
- name: config
configMap:
name: redis5-configmap
- name: probes
configMap:
name: redis-probes
volumeClaimTemplates:
- metadata:
name: data
annotations:
spec:
accessModes:
- "ReadWriteMany"
resources:
requests:
storage: "10Gi"
#修改成自己的存储类
storageClassName: longhorn三、执行配置文件
kubectl apply -f redis-cluster-sts.yaml四、查看节点集群状态 ，主从状态
[root@xm-nano-k8s-master-113-59 ]# kubectl exec -it -n prod redis5-sentinel-0 -- sh -c redis-cli
Defaulted container "redis" out of: redis, sentinel, config-init (init)
127.0.0.1:6379> info replication
# Replication
role:master
connected_slaves:2
min_slaves_good_slaves:2
slave0:ip=10.42.1.155,port=6379,state=online,offset=154978,lag=0
slave1:ip=10.42.7.26,port=6379,state=online,offset=154978,lag=0
master_replid:adecf662565b1dc90eca2af58e3ebbe3ce899b45
master_replid2:7e486d561259c04f0785e1250ef089683bdc4241
master_repl_offset:155114
second_repl_offset:84363
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:74662
repl_backlog_histlen:80453
验证：master节点删除后重启，查看master是否自动漂移。Pod被删除后会自动启动，且启动成功后会自动变为Slave节点.
集群内部程序连接哨兵的地址：
创建一个Headless Service（无头服务）来管理哨兵Pod，但不提供负载均衡和ClusterIP。Headless Service会为每个哨兵Pod分配一个DNS条目，格式为<pod-name>.<service-name>.<namespace>.svc.cluster.local。
例如：redis5-sentinel-2.redis5-sentinel-headless.prod.svc.cluster.local.:26379
集群外部访问K8s的redis哨兵(gpt版本)
在Kubernetes（k8s）环境中，由于Pod的IP地址是动态分配的，并且会随着Pod的重启或重新调度而改变，因此直接在外部程序的连接配置中硬编码哨兵节点的IP地址和端口是不可行的。然而，如果你的外部程序确实需要这种配置（尽管这通常不是最佳实践），你可以采取以下几种策略：
1. 使用NodePort或LoadBalancer类型的Service
虽然这不是直接提供哨兵节点IP地址的方法，但你可以为哨兵节点创建一个或多个NodePort或LoadBalancer类型的Service。然而，请注意，这通常不是哨兵节点的标准用法，因为哨兵的主要目的是在Redis主节点故障时自动进行故障转移，而不是直接对外提供服务。
不过，如果你确实需要这样做，你可以为每个哨兵节点（或一组哨兵节点）创建一个Service，并配置为NodePort或LoadBalancer类型。然后，你可以将Service的外部IP地址（对于LoadBalancer类型）或节点的IP地址加上NodePort端口（对于NodePort类型）提供给外部程序。但是，请注意，这将为每个哨兵节点提供一个可访问的端点，而不是一个统一的哨兵服务入口。
2. 使用DNS记录或静态IP地址（如果可用）
如果你的Kubernetes集群部署在支持静态IP地址或自定义DNS记录的环境中（如某些云提供商的托管Kubernetes服务），你可以考虑为哨兵节点创建静态IP地址或DNS记录。然而，这通常需要你与你的云提供商合作，并且可能涉及到额外的成本。
3. 使用Ingress（不推荐）
Ingress通常用于HTTP/HTTPS路由，而不是TCP/UDP服务（如Redis哨兵）。因此，使用Ingress来暴露哨兵节点可能不是一个好的选择，除非你的Ingress控制器支持TCP/UDP路由，并且你愿意为此进行额外的配置。
4. 编写一个中间服务
你可以编写一个中间服务（可能是一个简单的Web服务或API），该服务运行在Kubernetes集群内部或外部，并负责查询哨兵Pod的当前IP地址和端口。然后，你的外部程序可以调用这个中间服务来获取哨兵节点的地址信息。这种方法需要额外的开发和维护工作，但它提供了最大的灵活性。
5. 使用客户端库的支持
如果可能的话，最好的解决方案是更新你的外部程序或使用的客户端库，以支持通过哨兵服务发现Redis主节点。大多数现代Redis客户端库都支持哨兵模式，并可以自动处理哨兵节点的发现和故障转移。
参考链接 ：