fix(connector): fix redis cluster resource recovery

This commit is contained in:
Ilya Averyanov 2023-01-26 00:37:12 +02:00 committed by Ilya Averyanov
parent 6175076f6f
commit fce1e74c3d
26 changed files with 340 additions and 216 deletions

View File

@ -1,5 +1,5 @@
MYSQL_TAG=8
REDIS_TAG=6
REDIS_TAG=7.0
MONGO_TAG=5
PGSQL_TAG=13
LDAP_TAG=2.4.50

View File

@ -13,10 +13,10 @@ help:
up:
env \
MYSQL_TAG=8 \
REDIS_TAG=6 \
REDIS_TAG=7.0 \
MONGO_TAG=5 \
PGSQL_TAG=13 \
docker compose \
docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
@ -34,7 +34,7 @@ up:
up -d --build --remove-orphans
down:
docker compose \
docker-compose \
-f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \

View File

@ -1,11 +1,57 @@
version: '3.9'
services:
redis_cluster:
redis-cluster-1: &redis-node
container_name: redis-cluster-1
image: redis:${REDIS_TAG}
container_name: redis-cluster
volumes:
- ./redis/:/data/conf
command: bash -c "/bin/bash /data/conf/redis.sh --node cluster && tail -f /var/log/redis-server.log"
- ./redis/cluster-tcp:/usr/local/etc/redis
command: redis-server /usr/local/etc/redis/redis.conf
networks:
- emqx_bridge
redis-cluster-2:
<<: *redis-node
container_name: redis-cluster-2
redis-cluster-3:
<<: *redis-node
container_name: redis-cluster-3
redis-cluster-4:
<<: *redis-node
container_name: redis-cluster-4
redis-cluster-5:
<<: *redis-node
container_name: redis-cluster-5
redis-cluster-6:
<<: *redis-node
container_name: redis-cluster-6
redis-cluster-create:
<<: *redis-node
container_name: redis-cluster-create
command: >
redis-cli
--cluster create
redis-cluster-1:6379
redis-cluster-2:6379
redis-cluster-3:6379
redis-cluster-4:6379
redis-cluster-5:6379
redis-cluster-6:6379
--cluster-replicas 1
--cluster-yes
--pass "public"
--no-auth-warning
depends_on:
- redis-cluster-1
- redis-cluster-2
- redis-cluster-3
- redis-cluster-4
- redis-cluster-5
- redis-cluster-6

View File

@ -1,14 +1,59 @@
version: '3.9'
services:
redis_cluster_tls:
container_name: redis-cluster-tls
redis-cluster-tls-1: &redis-node
container_name: redis-cluster-tls-1
image: redis:${REDIS_TAG}
volumes:
- ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt
- ../../apps/emqx/etc/certs/cert.pem:/etc/certs/redis.crt
- ../../apps/emqx/etc/certs/key.pem:/etc/certs/redis.key
- ./redis/:/data/conf
command: bash -c "/bin/bash /data/conf/redis.sh --node cluster --tls-enabled && tail -f /var/log/redis-server.log"
- ./redis/cluster-tls:/usr/local/etc/redis
- ../../apps/emqx/etc/certs:/etc/certs
command: redis-server /usr/local/etc/redis/redis.conf
networks:
- emqx_bridge
redis-cluster-tls-2:
<<: *redis-node
container_name: redis-cluster-tls-2
redis-cluster-tls-3:
<<: *redis-node
container_name: redis-cluster-tls-3
redis-cluster-tls-4:
<<: *redis-node
container_name: redis-cluster-tls-4
redis-cluster-tls-5:
<<: *redis-node
container_name: redis-cluster-tls-5
redis-cluster-tls-6:
<<: *redis-node
container_name: redis-cluster-tls-6
redis-cluster-tls-create:
<<: *redis-node
container_name: redis-cluster-tls-create
command: >
redis-cli
--cluster create
redis-cluster-tls-1:6389
redis-cluster-tls-2:6389
redis-cluster-tls-3:6389
redis-cluster-tls-4:6389
redis-cluster-tls-5:6389
redis-cluster-tls-6:6389
--cluster-replicas 1
--cluster-yes
--pass "public"
--no-auth-warning
--tls
--insecure
depends_on:
- redis-cluster-tls-1
- redis-cluster-tls-2
- redis-cluster-tls-3
- redis-cluster-tls-4
- redis-cluster-tls-5
- redis-cluster-tls-6

View File

@ -1,11 +1,41 @@
version: '3.9'
version: "3"
services:
redis_sentinel_server:
redis-sentinel-master:
container_name: redis-sentinel-master
image: redis:${REDIS_TAG}
volumes:
- ./redis/sentinel-tcp:/usr/local/etc/redis
command: redis-server /usr/local/etc/redis/master.conf
networks:
- emqx_bridge
redis-sentinel-slave:
container_name: redis-sentinel-slave
image: redis:${REDIS_TAG}
volumes:
- ./redis/sentinel-tcp:/usr/local/etc/redis
command: redis-server /usr/local/etc/redis/slave.conf
networks:
- emqx_bridge
depends_on:
- redis-sentinel-master
redis-sentinel:
container_name: redis-sentinel
image: redis:${REDIS_TAG}
volumes:
- ./redis/:/data/conf
command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel && tail -f /var/log/redis-server.log"
- ./redis/sentinel-tcp/sentinel-base.conf:/usr/local/etc/redis/sentinel-base.conf
depends_on:
- redis-sentinel-master
- redis-sentinel-slave
command: >
bash -c "cp -f /usr/local/etc/redis/sentinel-base.conf /usr/local/etc/redis/sentinel.conf &&
redis-sentinel /usr/local/etc/redis/sentinel.conf"
networks:
- emqx_bridge

View File

@ -1,14 +1,44 @@
version: '3.9'
version: "3"
services:
redis_sentinel_server_tls:
redis-sentinel-tls-master:
container_name: redis-sentinel-tls-master
image: redis:${REDIS_TAG}
volumes:
- ./redis/sentinel-tls:/usr/local/etc/redis
- ../../apps/emqx/etc/certs:/etc/certs
command: redis-server /usr/local/etc/redis/master.conf
networks:
- emqx_bridge
redis-sentinel-tls-slave:
container_name: redis-sentinel-tls-slave
image: redis:${REDIS_TAG}
volumes:
- ./redis/sentinel-tls:/usr/local/etc/redis
- ../../apps/emqx/etc/certs:/etc/certs
command: redis-server /usr/local/etc/redis/slave.conf
networks:
- emqx_bridge
depends_on:
- redis-sentinel-tls-master
redis-sentinel-tls:
container_name: redis-sentinel-tls
image: redis:${REDIS_TAG}
volumes:
- ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.crt
- ../../apps/emqx/etc/certs/cert.pem:/etc/certs/redis.crt
- ../../apps/emqx/etc/certs/key.pem:/etc/certs/redis.key
- ./redis/:/data/conf
command: bash -c "/bin/bash /data/conf/redis.sh --node sentinel --tls-enabled && tail -f /var/log/redis-server.log"
- ./redis/sentinel-tls/sentinel-base.conf:/usr/local/etc/redis/sentinel-base.conf
- ../../apps/emqx/etc/certs:/etc/certs
depends_on:
- redis-sentinel-tls-master
- redis-sentinel-tls-slave
command: >
bash -c "cp -f /usr/local/etc/redis/sentinel-base.conf /usr/local/etc/redis/sentinel.conf &&
redis-sentinel /usr/local/etc/redis/sentinel.conf"
networks:
- emqx_bridge

View File

@ -1,3 +0,0 @@
r700?i.log
nodes.700?.conf
*.rdb

View File

@ -0,0 +1,18 @@
bind :: 0.0.0.0
port 6379
requirepass public
cluster-enabled yes
masterauth public
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -0,0 +1,28 @@
bind :: 0.0.0.0
port 6379
requirepass public
cluster-enabled yes
masterauth public
tls-port 6389
tls-cert-file /etc/certs/cert.pem
tls-key-file /etc/certs/key.pem
tls-ca-cert-file /etc/certs/cacert.pem
tls-auth-clients no
tls-replication yes
tls-cluster yes
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -1,12 +0,0 @@
daemonize yes
bind 0.0.0.0 ::
logfile /var/log/redis-server.log
protected-mode no
requirepass public
masterauth public
tls-cert-file /etc/certs/redis.crt
tls-key-file /etc/certs/redis.key
tls-ca-cert-file /etc/certs/ca.crt
tls-replication yes
tls-cluster yes

View File

@ -1,6 +0,0 @@
daemonize yes
bind 0.0.0.0 ::
logfile /var/log/redis-server.log
protected-mode no
requirepass public
masterauth public

View File

@ -1,126 +0,0 @@
#!/bin/bash
set -x
LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1)
node=single
tls=false
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-n|--node)
node="$2"
shift # past argument
shift # past value
;;
--tls-enabled)
tls=true
shift # past argument
;;
*)
shift # past argument
;;
esac
done
rm -f \
/data/conf/r7000i.log \
/data/conf/r7001i.log \
/data/conf/r7002i.log \
/data/conf/nodes.7000.conf \
/data/conf/nodes.7001.conf \
/data/conf/nodes.7002.conf
if [ "$node" = "cluster" ]; then
if $tls; then
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--tls-port 8000 --cluster-enabled yes
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--tls-port 8001 --cluster-enabled yes
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--tls-port 8002 --cluster-enabled yes
else
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--cluster-enabled yes
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--cluster-enabled yes
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--cluster-enabled yes
fi
elif [ "$node" = "sentinel" ]; then
if $tls; then
redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--tls-port 8000 --cluster-enabled no
redis-server /data/conf/redis-tls.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--tls-port 8001 --cluster-enabled no --slaveof "$LOCAL_IP" 8000
redis-server /data/conf/redis-tls.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--tls-port 8002 --cluster-enabled no --slaveof "$LOCAL_IP" 8000
else
redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
--cluster-enabled no
redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
--cluster-enabled no --slaveof "$LOCAL_IP" 7000
redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf \
--cluster-enabled no --slaveof "$LOCAL_IP" 7000
fi
fi
REDIS_LOAD_FLG=true
while $REDIS_LOAD_FLG;
do
sleep 1
redis-cli --pass public --no-auth-warning -p 7000 info 1> /data/conf/r7000i.log 2> /dev/null
if ! [ -s /data/conf/r7000i.log ]; then
continue
fi
redis-cli --pass public --no-auth-warning -p 7001 info 1> /data/conf/r7001i.log 2> /dev/null
if ! [ -s /data/conf/r7001i.log ]; then
continue
fi
redis-cli --pass public --no-auth-warning -p 7002 info 1> /data/conf/r7002i.log 2> /dev/null;
if ! [ -s /data/conf/r7002i.log ]; then
continue
fi
if [ "$node" = "cluster" ] ; then
if $tls; then
yes "yes" | redis-cli --cluster create "$LOCAL_IP:8000" "$LOCAL_IP:8001" "$LOCAL_IP:8002" \
--pass public --no-auth-warning \
--tls true --cacert /etc/certs/ca.crt \
--cert /etc/certs/redis.crt --key /etc/certs/redis.key
else
yes "yes" | redis-cli --cluster create "$LOCAL_IP:7000" "$LOCAL_IP:7001" "$LOCAL_IP:7002" \
--pass public --no-auth-warning
fi
elif [ "$node" = "sentinel" ]; then
tee /_sentinel.conf>/dev/null << EOF
port 26379
bind 0.0.0.0 ::
daemonize yes
logfile /var/log/redis-server.log
dir /tmp
EOF
if $tls; then
cat >>/_sentinel.conf<<EOF
tls-port 26380
tls-replication yes
tls-cert-file /etc/certs/redis.crt
tls-key-file /etc/certs/redis.key
tls-ca-cert-file /etc/certs/ca.crt
sentinel monitor mymaster $LOCAL_IP 8000 1
EOF
else
cat >>/_sentinel.conf<<EOF
sentinel monitor mymaster $LOCAL_IP 7000 1
EOF
fi
redis-server /_sentinel.conf --sentinel
fi
REDIS_LOAD_FLG=false
done
exit 0;

View File

@ -0,0 +1,14 @@
bind :: 0.0.0.0
port 6379
requirepass public
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -0,0 +1,7 @@
sentinel resolve-hostnames yes
bind :: 0.0.0.0
sentinel monitor mymaster redis-sentinel-master 6379 1
sentinel auth-pass mymaster public
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 20000

View File

@ -0,0 +1,17 @@
bind :: 0.0.0.0
port 6379
requirepass public
replicaof redis-sentinel-master 6379
masterauth public
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -0,0 +1,20 @@
bind :: 0.0.0.0
port 6379
requirepass public
tls-port 6389
tls-cert-file /etc/certs/cert.pem
tls-key-file /etc/certs/key.pem
tls-ca-cert-file /etc/certs/cacert.pem
tls-auth-clients no
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -0,0 +1,14 @@
sentinel resolve-hostnames yes
bind :: 0.0.0.0
tls-port 26380
tls-replication yes
tls-cert-file /etc/certs/cert.pem
tls-key-file /etc/certs/key.pem
tls-ca-cert-file /etc/certs/cacert.pem
tls-auth-clients no
sentinel monitor mymaster redis-sentinel-tls-master 6389 1
sentinel auth-pass mymaster public
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 20000

View File

@ -0,0 +1,24 @@
bind :: 0.0.0.0
port 6379
requirepass public
replicaof redis-sentinel-tls-master 6389
masterauth public
tls-port 6389
tls-replication yes
tls-cert-file /etc/certs/cert.pem
tls-key-file /etc/certs/key.pem
tls-ca-cert-file /etc/certs/cacert.pem
tls-auth-clients no
protected-mode no
daemonize no
loglevel notice
logfile ""
always-show-logo no
save ""
appendonly no

View File

@ -155,11 +155,11 @@ jobs:
working-directory: source
env:
DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04"
MONGO_TAG: 5
MYSQL_TAG: 8
PGSQL_TAG: 13
REDIS_TAG: 6
INFLUXDB_TAG: 2.5.0
MONGO_TAG: "5"
MYSQL_TAG: "8"
PGSQL_TAG: "13"
REDIS_TAG: "7.0"
INFLUXDB_TAG: "2.5.0"
PROFILE: ${{ matrix.profile }}
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}

View File

@ -1,4 +1,5 @@
mongo
redis
redis_cluster
mysql
pgsql

View File

@ -11,17 +11,9 @@
{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
{mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
{epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}},
%% NOTE: mind poolboy version when updating mongodb-erlang version
{mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}},
%% NOTE: mind poolboy version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.7.5"}}},
%% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
%% (which has overflow_ttl feature added).
%% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07).
%% By accident, We have always been using the upstream fork due to
%% eredis_cluster's dependency getting resolved earlier.
%% Here we pin 1.5.2 to avoid surprises in the future.
{poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}
%% NOTE: mind ecpool version when updating eredis_cluster version
{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}}
]}.
{shell, [

View File

@ -153,7 +153,7 @@ on_start(
false ->
[{ssl, false}]
end ++ [{sentinel, maps:get(sentinel, Config, undefined)}],
PoolName = emqx_plugin_libs_pool:pool_name(InstId),
PoolName = InstId,
State = #{poolname => PoolName, type => Type},
case Type of
cluster ->
@ -225,26 +225,10 @@ is_unrecoverable_error({error, <<"ERR unknown command ", _/binary>>}) ->
is_unrecoverable_error(_) ->
false.
extract_eredis_cluster_workers(PoolName) ->
lists:flatten([
gen_server:call(PoolPid, get_all_workers)
|| PoolPid <- eredis_cluster_monitor:get_all_pools(PoolName)
]).
eredis_cluster_workers_exist_and_are_connected(Workers) ->
length(Workers) > 0 andalso
lists:all(
fun({_, Pid, _, _}) ->
eredis_cluster_pool_worker:is_connected(Pid) =:= true
end,
Workers
).
on_get_status(_InstId, #{type := cluster, poolname := PoolName}) ->
case eredis_cluster:pool_exists(PoolName) of
true ->
Workers = extract_eredis_cluster_workers(PoolName),
Health = eredis_cluster_workers_exist_and_are_connected(Workers),
Health = eredis_cluster:ping_all(PoolName),
status_result(Health);
false ->
disconnected

View File

@ -27,6 +27,8 @@
-define(REDIS_SINGLE_PORT, 6379).
-define(REDIS_SENTINEL_HOST, "redis-sentinel").
-define(REDIS_SENTINEL_PORT, 26379).
-define(REDIS_CLUSTER_HOST, "redis-cluster-1").
-define(REDIS_CLUSTER_PORT, 6379).
-define(REDIS_RESOURCE_MOD, emqx_connector_redis).
all() ->
@ -203,8 +205,8 @@ redis_config_base(Type, ServerKey) ->
MaybeSentinel = "",
MaybeDatabase = " database = 1\n";
"cluster" ->
Host = ?REDIS_SINGLE_HOST,
Port = ?REDIS_SINGLE_PORT,
Host = ?REDIS_CLUSTER_HOST,
Port = ?REDIS_CLUSTER_PORT,
MaybeSentinel = "",
MaybeDatabase = ""
end,

View File

@ -479,12 +479,13 @@ redis_connect_configs() ->
},
redis_cluster => #{
tcp => #{
<<"servers">> => <<"redis-cluster:7000,redis-cluster:7001,redis-cluster:7002">>,
<<"servers">> =>
<<"redis-cluster-1:6379,redis-cluster-2:6379,redis-cluster-3:6379">>,
<<"redis_type">> => <<"cluster">>
},
tls => #{
<<"servers">> =>
<<"redis-cluster-tls:8000,redis-cluster-tls:8001,redis-cluster-tls:8002">>,
<<"redis-cluster-tls-1:6389,redis-cluster-tls-2:6389,redis-cluster-tls-3:6389">>,
<<"redis_type">> => <<"cluster">>,
<<"ssl">> => redis_connect_ssl_opts(redis_cluster)
}

View File

@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.7", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.2", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
{:replayq, github: "emqx/replayq", tag: "0.3.6", override: true},
{:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
{:emqtt, github: "emqx/emqtt", tag: "1.7.0-rc.2", override: true},
@ -76,8 +76,6 @@ defmodule EMQXUmbrella.MixProject do
{:gun, github: "emqx/gun", tag: "1.3.9", override: true},
# in conflict by emqx_connector and system_monitor
{:epgsql, github: "emqx/epgsql", tag: "4.7.0.1", override: true},
# in conflict by mongodb and eredis_cluster
{:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true},
# in conflict by emqx and observer_cli
{:recon, github: "ferd/recon", tag: "2.5.1", override: true},
{:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},

View File

@ -59,7 +59,7 @@
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
, {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.6"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0-rc.2"}}}