From b472b56883521f2ced95e9f827efd23fcdbcce29 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 16 Jan 2024 17:09:24 +0200 Subject: [PATCH 01/32] perf(emqx_cm): use a dedicated pool for channel cleanup This is to isolate channels cleanup from other async tasks (like routes cleanup), as channels cleanup can be quite slow under high network latency conditions. Fixes: EMQX-11743 --- apps/emqx/include/emqx_cm.hrl | 2 + apps/emqx/src/emqx_cm.erl | 6 ++- apps/emqx/src/emqx_cm_sup.erl | 4 ++ apps/emqx/src/emqx_pool.erl | 64 +++++++++++++++++++++++--------- apps/emqx/test/emqx_cm_SUITE.erl | 10 +++-- changes/ce/perf-12336.en.md | 2 + 6 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 changes/ce/perf-12336.en.md diff --git a/apps/emqx/include/emqx_cm.hrl b/apps/emqx/include/emqx_cm.hrl index ae70f131f..6478a6162 100644 --- a/apps/emqx/include/emqx_cm.hrl +++ b/apps/emqx/include/emqx_cm.hrl @@ -30,4 +30,6 @@ -define(T_GET_INFO, 5_000). -define(T_TAKEOVER, 15_000). +-define(CM_POOL, emqx_cm_pool). + -endif. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 660ac3cfe..2e6714e7f 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -670,7 +670,11 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon} ChanPids = [Pid | emqx_utils:drain_down(BatchSize)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), - ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), + ok = emqx_pool:async_submit_to_pool( + ?CM_POOL, + fun lists:foreach/2, + [fun ?MODULE:clean_down/1, Items] + ), {noreply, State#{chan_pmon := PMon1}}; handle_info(Info, State) -> ?SLOG(error, #{msg => "unexpected_info", info => Info}), diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index e7420b4da..622921f1d 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -25,6 +25,8 @@ %% for test -export([restart_flapping/0]). +-include("emqx_cm.hrl"). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -45,6 +47,7 @@ init([]) -> Banned = child_spec(emqx_banned, 1000, worker), Flapping = child_spec(emqx_flapping, 1000, worker), Locker = child_spec(emqx_cm_locker, 5000, worker), + CmPool = emqx_pool_sup:spec(emqx_cm_pool_sup, [?CM_POOL, random, {emqx_pool, start_link, []}]), Registry = child_spec(emqx_cm_registry, 5000, worker), Manager = child_spec(emqx_cm, 5000, worker), DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor), @@ -53,6 +56,7 @@ init([]) -> Banned, Flapping, Locker, + CmPool, Registry, Manager, DSSessionGCSup diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 1cb5f429c..39c585133 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -28,11 +28,15 @@ submit/1, submit/2, async_submit/1, - async_submit/2 + async_submit/2, + submit_to_pool/2, + submit_to_pool/3, + async_submit_to_pool/2, + async_submit_to_pool/3 ]). -ifdef(TEST). --export([worker/0, flush_async_tasks/0]). +-export([worker/0, flush_async_tasks/0, flush_async_tasks/1]). -endif. %% gen_server callbacks @@ -57,7 +61,7 @@ -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_utils:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(Pool, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] @@ -66,32 +70,48 @@ start_link(Pool, Id) -> %% @doc Submit work to the pool. -spec submit(task()) -> any(). submit(Task) -> - call({submit, Task}). + submit_to_pool(?POOL, Task). -spec submit(fun(), list(any())) -> any(). submit(Fun, Args) -> - call({submit, {Fun, Args}}). - -%% @private -call(Req) -> - gen_server:call(worker(), Req, infinity). + submit_to_pool(?POOL, Fun, Args). %% @doc Submit work to the pool asynchronously. -spec async_submit(task()) -> ok. async_submit(Task) -> - cast({async_submit, Task}). + async_submit_to_pool(?POOL, Task). -spec async_submit(fun(), list(any())) -> ok. async_submit(Fun, Args) -> - cast({async_submit, {Fun, Args}}). + async_submit_to_pool(?POOL, Fun, Args). + +-spec submit_to_pool(any(), task()) -> any(). +submit_to_pool(Pool, Task) -> + call(Pool, {submit, Task}). + +-spec submit_to_pool(any(), fun(), list(any())) -> any(). +submit_to_pool(Pool, Fun, Args) -> + call(Pool, {submit, {Fun, Args}}). + +-spec async_submit_to_pool(any(), task()) -> ok. +async_submit_to_pool(Pool, Task) -> + cast(Pool, {async_submit, Task}). + +-spec async_submit_to_pool(any(), fun(), list(any())) -> ok. +async_submit_to_pool(Pool, Fun, Args) -> + cast(Pool, {async_submit, {Fun, Args}}). %% @private -cast(Msg) -> - gen_server:cast(worker(), Msg). +call(Pool, Req) -> + gen_server:call(worker(Pool), Req, infinity). %% @private -worker() -> - gproc_pool:pick_worker(?POOL). +cast(Pool, Msg) -> + gen_server:cast(worker(Pool), Msg). + +%% @private +worker(Pool) -> + gproc_pool:pick_worker(Pool). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -146,15 +166,25 @@ run(Fun) when is_function(Fun) -> Fun(). -ifdef(TEST). + +worker() -> + worker(?POOL). + +flush_async_tasks() -> + flush_async_tasks(?POOL). + %% This help function creates a large enough number of async tasks %% to force flush the pool workers. %% The number of tasks should be large enough to ensure all workers have %% the chance to work on at least one of the tasks. -flush_async_tasks() -> +flush_async_tasks(Pool) -> Ref = make_ref(), Self = self(), L = lists:seq(1, 997), - lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, Ref, I} end, []) end, L), + lists:foreach( + fun(I) -> emqx_pool:async_submit_to_pool(Pool, fun() -> Self ! {done, Ref, I} end, []) end, + L + ), lists:foreach( fun(I) -> receive diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 4ecea9a4b..e175b4349 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -221,7 +221,7 @@ t_open_session_race_condition(_) -> end, %% sync ignored = gen_server:call(?CM, ignore, infinity), - ok = emqx_pool:flush_async_tasks(), + ok = emqx_pool:flush_async_tasks(?CM_POOL), ?assertEqual([], emqx_cm:lookup_channels(ClientId)). t_kick_session_discard_normal(_) -> @@ -343,7 +343,7 @@ test_stepdown_session(Action, Reason) -> end, % sync ignored = gen_server:call(?CM, ignore, infinity), - ok = flush_emqx_pool(), + ok = flush_emqx_cm_pool(), ?assertEqual([], emqx_cm:lookup_channels(ClientId)). %% Channel deregistration is delegated to emqx_pool as a sync tasks. @@ -353,10 +353,12 @@ test_stepdown_session(Action, Reason) -> %% to sync with the pool workers. %% The number of tasks should be large enough to ensure all workers have %% the chance to work on at least one of the tasks. -flush_emqx_pool() -> +flush_emqx_cm_pool() -> Self = self(), L = lists:seq(1, 1000), - lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, I} end, []) end, L), + lists:foreach( + fun(I) -> emqx_pool:async_submit_to_pool(?CM_POOL, fun() -> Self ! {done, I} end, []) end, L + ), lists:foreach( fun(I) -> receive diff --git a/changes/ce/perf-12336.en.md b/changes/ce/perf-12336.en.md new file mode 100644 index 000000000..5c385e6b6 --- /dev/null +++ b/changes/ce/perf-12336.en.md @@ -0,0 +1,2 @@ +Isolate channels cleanup from other async tasks (like routes cleanup) by using a dedicated pool, +as this task can be quite slow under high network latency conditions. From 80e82db28249544df5f4e720a66014e0fc47f983 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 16 Jan 2024 19:42:37 +0200 Subject: [PATCH 02/32] test(emqx_cm_SUITE): use one helper function: `emqx_pool:flush_async_tasks/1` --- apps/emqx/test/emqx_cm_SUITE.erl | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index e175b4349..aba4bc744 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -343,31 +343,9 @@ test_stepdown_session(Action, Reason) -> end, % sync ignored = gen_server:call(?CM, ignore, infinity), - ok = flush_emqx_cm_pool(), + ok = emqx_pool:flush_async_tasks(?CM_POOL), ?assertEqual([], emqx_cm:lookup_channels(ClientId)). -%% Channel deregistration is delegated to emqx_pool as a sync tasks. -%% The emqx_pool is pool of workers, and there is no way to know -%% which worker was picked for the last deregistration task. -%% This help function creates a large enough number of async tasks -%% to sync with the pool workers. -%% The number of tasks should be large enough to ensure all workers have -%% the chance to work on at least one of the tasks. -flush_emqx_cm_pool() -> - Self = self(), - L = lists:seq(1, 1000), - lists:foreach( - fun(I) -> emqx_pool:async_submit_to_pool(?CM_POOL, fun() -> Self ! {done, I} end, []) end, L - ), - lists:foreach( - fun(I) -> - receive - {done, I} -> ok - end - end, - L - ). - t_discard_session_race(_) -> ClientId = rand_client_id(), ?check_trace( From 6a0c54b40e15a7e99c7a67ae6448c54b9e254859 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 17:31:28 -0300 Subject: [PATCH 03/32] ci(dev): add `.git/` to `.gitignore` For some reason, some tools like [ripgrep](https://github.com/BurntSushi/ripgrep) will search `.git` when using the `-.`/`--hidden` flag, even when not using `--no-ignore-vcs`. This leads to several unwanted results. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a76c3807..7068c1c7d 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ apps/emqx_conf/etc/emqx.conf.all.rendered* rebar-git-cache.tar # build docker image locally .docker_image_tag +.git/ From 0a0bf3123bfbd97cc473f95feb8a97808d3f9a07 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 17 Jan 2024 15:18:03 +0100 Subject: [PATCH 04/32] ci: bump actions versions --- .github/actions/package-macos/action.yaml | 2 +- .github/workflows/_pr_entrypoint.yaml | 2 +- .github/workflows/_push-entrypoint.yaml | 2 +- .github/workflows/build_docker_for_test.yaml | 2 +- .github/workflows/build_packages.yaml | 2 +- .github/workflows/build_packages_cron.yaml | 4 ++-- .github/workflows/build_slim_packages.yaml | 6 +++--- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/performance_test.yaml | 18 +++++++++--------- .github/workflows/run_conf_tests.yaml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 2 +- .github/workflows/run_jmeter_tests.yaml | 14 +++++++------- .github/workflows/run_relup_tests.yaml | 4 ++-- .github/workflows/run_test_cases.yaml | 10 +++++----- .github/workflows/scorecard.yaml | 2 +- .github/workflows/static_checks.yaml | 2 +- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index bae335cf0..64d179b46 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -51,7 +51,7 @@ runs: echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT ;; esac - - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 id: cache if: steps.prepare.outputs.SELF_HOSTED != 'true' with: diff --git a/.github/workflows/_pr_entrypoint.yaml b/.github/workflows/_pr_entrypoint.yaml index 86e676ebe..1f7e8e466 100644 --- a/.github/workflows/_pr_entrypoint.yaml +++ b/.github/workflows/_pr_entrypoint.yaml @@ -144,7 +144,7 @@ jobs: echo "PROFILE=${PROFILE}" | tee -a .env echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: ${{ matrix.profile }} path: ${{ matrix.profile }}.zip diff --git a/.github/workflows/_push-entrypoint.yaml b/.github/workflows/_push-entrypoint.yaml index 1e0dd941b..a6d0e178e 100644 --- a/.github/workflows/_push-entrypoint.yaml +++ b/.github/workflows/_push-entrypoint.yaml @@ -152,7 +152,7 @@ jobs: echo "PROFILE=${PROFILE}" | tee -a .env echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip . - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: ${{ matrix.profile }} path: ${{ matrix.profile }}.zip diff --git a/.github/workflows/build_docker_for_test.yaml b/.github/workflows/build_docker_for_test.yaml index ccff642f9..25adea083 100644 --- a/.github/workflows/build_docker_for_test.yaml +++ b/.github/workflows/build_docker_for_test.yaml @@ -57,7 +57,7 @@ jobs: - name: export docker image run: | docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: "${{ env.EMQX_NAME }}-docker" path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 3382bbeed..abde2672e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -95,7 +95,7 @@ jobs: apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: ${{ matrix.profile }}-${{ matrix.otp }}-${{ matrix.os }} diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 56d5c37f2..5e90be8c4 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -66,7 +66,7 @@ jobs: set -eu ./scripts/pkg-tests.sh "${PROFILE}-tgz" ./scripts/pkg-tests.sh "${PROFILE}-pkg" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: ${{ matrix.profile[0] }}-${{ matrix.os }} @@ -111,7 +111,7 @@ jobs: apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: ${{ matrix.profile }}-${{ matrix.os }} diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 4b9ca76b9..45dee2b3d 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -88,13 +88,13 @@ jobs: run: | make ${EMQX_NAME}-elixir-pkg ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}" path: _packages/${{ matrix.profile[0] }}/* retention-days: 7 compression-level: 0 - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}" path: | @@ -128,7 +128,7 @@ jobs: apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: ${{ matrix.os }} path: _packages/**/* diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 30d788500..cfe6cfbae 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -36,7 +36,7 @@ jobs: MIX_ENV: emqx-enterprise PROFILE: emqx-enterprise - name: Upload produced lock files - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: produced_lock_files diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index ede8abf07..629e8fcdb 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -52,7 +52,7 @@ jobs: id: package_file run: | echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: emqx-ubuntu20.04 path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }} @@ -113,13 +113,13 @@ jobs: working-directory: ./tf-emqx-performance-test run: | terraform destroy -auto-approve - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: metrics path: | "./tf-emqx-performance-test/*.tar.gz" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: terraform @@ -184,13 +184,13 @@ jobs: working-directory: ./tf-emqx-performance-test run: | terraform destroy -auto-approve - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: metrics path: | "./tf-emqx-performance-test/*.tar.gz" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: terraform @@ -257,13 +257,13 @@ jobs: working-directory: ./tf-emqx-performance-test run: | terraform destroy -auto-approve - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: metrics path: | "./tf-emqx-performance-test/*.tar.gz" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: terraform @@ -330,13 +330,13 @@ jobs: working-directory: ./tf-emqx-performance-test run: | terraform destroy -auto-approve - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: success() with: name: metrics path: | "./tf-emqx-performance-test/*.tar.gz" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: terraform diff --git a/.github/workflows/run_conf_tests.yaml b/.github/workflows/run_conf_tests.yaml index cac63910b..913f4e5a4 100644 --- a/.github/workflows/run_conf_tests.yaml +++ b/.github/workflows/run_conf_tests.yaml @@ -40,7 +40,7 @@ jobs: if: failure() run: | cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.* - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: conftest-logs-${{ matrix.profile }} diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index f7c645aeb..e6326b96c 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -58,7 +58,7 @@ jobs: ./rebar3 eunit -v --name 'eunit@127.0.0.1' ./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true ./rebar3 proper -d test/props - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: logs-emqx-app-tests diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index 86cbf220f..14ee999ef 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -16,7 +16,7 @@ jobs: steps: - name: Cache Jmeter id: cache-jmeter - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 with: path: /tmp/apache-jmeter.tgz key: apache-jmeter-5.4.3.tgz @@ -35,7 +35,7 @@ jobs: else wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz $ARCHIVE_URL fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: apache-jmeter.tgz path: /tmp/apache-jmeter.tgz @@ -86,7 +86,7 @@ jobs: echo "check logs failed" exit 1 fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: always() with: name: jmeter_logs-advanced_feat-${{ matrix.scripts_type }} @@ -153,7 +153,7 @@ jobs: if: failure() run: | docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: always() with: name: jmeter_logs-pgsql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.pgsql_tag }} @@ -213,7 +213,7 @@ jobs: echo "check logs failed" exit 1 fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: always() with: name: jmeter_logs-mysql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.mysql_tag }} @@ -265,7 +265,7 @@ jobs: echo "check logs failed" exit 1 fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: always() with: name: jmeter_logs-JWT_authn-${{ matrix.scripts_type }} @@ -309,7 +309,7 @@ jobs: echo "check logs failed" exit 1 fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: always() with: name: jmeter_logs-built_in_database_authn_authz-${{ matrix.scripts_type }} diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index db8cef69d..b5016d71c 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -45,7 +45,7 @@ jobs: run: | export PROFILE='emqx-enterprise' make emqx-enterprise-tgz - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 name: Upload built emqx and test scenario with: name: relup_tests_emqx_built @@ -111,7 +111,7 @@ jobs: docker logs node2.emqx.io | tee lux_logs/emqx2.log exit 1 fi - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 name: Save debug data if: failure() with: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8841c845b..ca478a381 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -64,7 +64,7 @@ jobs: CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: make proper - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: coverdata-${{ matrix.profile }}-${{ matrix.otp }} path: _build/test/cover @@ -108,7 +108,7 @@ jobs: ENABLE_COVER_COMPILE: 1 CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} path: _build/test/cover @@ -116,7 +116,7 @@ jobs: - name: compress logs if: failure() run: tar -czf logs.tar.gz _build/test/logs - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} @@ -155,7 +155,7 @@ jobs: CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} run: | make "${{ matrix.app }}-ct" - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} path: _build/test/cover @@ -164,7 +164,7 @@ jobs: - name: compress logs if: failure() run: tar -czf logs.tar.gz _build/test/logs - - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + - uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 if: failure() with: name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }} diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index f43892d01..aabe4e5b0 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -40,7 +40,7 @@ jobs: publish_results: true - name: "Upload artifact" - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: SARIF file path: results.sarif diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index a092210c8..96d3e31e9 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -37,7 +37,7 @@ jobs: run: | unzip -o -q ${{ matrix.profile }}.zip git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 with: path: "emqx_dialyzer_${{ matrix.otp }}_plt" key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }} From 74bf4042c529af52fc2b6bb1563e4afeaa22608a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 17 Jan 2024 22:44:51 +0300 Subject: [PATCH 05/32] fix(mqtt_bridge): render valid messages from incomplete rule data --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_msg.erl | 23 +++++++++---- .../test/emqx_bridge_mqtt_SUITE.erl | 34 +++++++++++++++++++ changes/ce/fix-12347.en.md | 4 +++ 4 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 changes/ce/fix-12347.en.md diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e6fe78ab8..0c00a0d59 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl index 48cae70d7..e09866429 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl @@ -16,6 +16,8 @@ -module(emqx_bridge_mqtt_msg). +-include_lib("emqx/include/emqx_mqtt.hrl"). + -export([parse/1]). -export([render/2]). @@ -66,8 +68,8 @@ render( #{ topic => render_string(TopicToken, Msg), payload => render_payload(Vars, Msg), - qos => render_simple_var(QoSToken, Msg), - retain => render_simple_var(RetainToken, Msg) + qos => render_simple_var(QoSToken, Msg, ?QOS_0), + retain => render_simple_var(RetainToken, Msg, false) }. render_payload(From, MapMsg) -> @@ -80,16 +82,23 @@ do_render_payload(Tks, Msg) -> %% Replace a string contains vars to another string in which the placeholders are replace by the %% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: -%% "a: 1". +%% "a: 1". Undefined vars will be replaced by empty strings. render_string(Tokens, Data) when is_list(Tokens) -> - emqx_placeholder:proc_tmpl(Tokens, Data, #{return => full_binary}); + emqx_placeholder:proc_tmpl(Tokens, Data, #{ + return => full_binary, var_trans => fun undefined_as_empty/1 + }); render_string(Val, _Data) -> Val. +undefined_as_empty(undefined) -> + <<>>; +undefined_as_empty(Val) -> + emqx_utils_conv:bin(Val). + %% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result %% value will be an integer 1. -render_simple_var(Tokens, Data) when is_list(Tokens) -> +render_simple_var(Tokens, Data, Default) when is_list(Tokens) -> [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), - Var; -render_simple_var(Val, _Data) -> + emqx_maybe:define(Var, Default); +render_simple_var(Val, _Data, _Default) -> Val. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6d1ff0915..46788b416 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -839,6 +839,40 @@ t_egress_mqtt_bridge_with_rules(_) -> {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []). +t_egress_mqtt_bridge_with_dummy_rule(_) -> + BridgeIDEgress = create_bridge( + ?SERVER_CONF#{ + <<"name">> => ?BRIDGE_NAME_EGRESS, + <<"egress">> => ?EGRESS_CONF + } + ), + + {ok, 201, Rule} = request( + post, + uri(["rules"]), + #{ + <<"name">> => <<"A_rule_send_empty_messages_to_a_sink_mqtt_bridge">>, + <<"enable">> => true, + <<"actions">> => [BridgeIDEgress], + %% select something useless from what a message cannot be composed + <<"sql">> => <<"SELECT x from \"t/1\"">> + } + ), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), + + %% PUBLISH a message to the rule. + Payload = <<"hi">>, + RuleTopic = <<"t/1">>, + RemoteTopic = <>, + emqx:subscribe(RemoteTopic), + timer:sleep(100), + emqx:publish(emqx_message:make(RuleTopic, Payload)), + %% we should receive a message on the "remote" broker, with specified topic + assert_mqtt_msg_received(RemoteTopic, <<>>), + + {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []). + t_mqtt_conn_bridge_egress_reconnect(_) -> %% then we add a mqtt connector, using POST BridgeIDEgress = create_bridge( diff --git a/changes/ce/fix-12347.en.md b/changes/ce/fix-12347.en.md new file mode 100644 index 000000000..b10738192 --- /dev/null +++ b/changes/ce/fix-12347.en.md @@ -0,0 +1,4 @@ +Always render valid messages for egress MQTT data bridge from the data fetched by Rule SQL, even if the data is incomplete and placeholders used in the bridge configuration are missing. +Previously, some messages were rendered as invalid and were discarded by the MQTT egress data bridge. + +Render undefined variables as empty strings in `payload` and `topic` templates of the MQTT egress data bridge. Previously, undefined variables were rendered as `undefined` strings. From 5b064e399f4df29358276fb011c1cf73a6b6c8ac Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 18 Jan 2024 17:29:23 +0800 Subject: [PATCH 06/32] chore: improve http connector logs format --- .../src/emqx_bridge_http_connector.erl | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 51375fc04..44c786751 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -251,6 +251,7 @@ start_pool(PoolName, PoolOpts) -> {error, {already_started, _}} -> ?SLOG(warning, #{ msg => "emqx_connector_on_start_already_started", + connector => PoolName, pool_name => PoolName }), ok; @@ -507,8 +508,8 @@ resolve_pool_worker(#{pool_name := PoolName} = State, Key) -> on_get_channels(ResId) -> emqx_bridge_v2:get_channels_for_connector(ResId). -on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) -> - case do_get_status(PoolName, Timeout) of +on_get_status(InstId, #{pool_name := InstId, connect_timeout := Timeout} = State) -> + case do_get_status(InstId, Timeout) of ok -> connected; {error, still_connecting} -> @@ -524,12 +525,7 @@ do_get_status(PoolName, Timeout) -> case ehttpc:health_check(Worker, Timeout) of ok -> ok; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "http_connector_get_status_failed", - reason => redact(Reason), - worker => Worker - }), + {error, _} = Error -> Error end end, @@ -540,14 +536,20 @@ do_get_status(PoolName, Timeout) -> case [E || {error, _} = E <- Results] of [] -> ok; - Errors -> - hd(Errors) + [{error, Reason} | _] -> + ?SLOG(info, #{ + msg => "health_check_failed", + reason => redact(Reason), + connector => PoolName + }), + {error, Reason} end catch exit:timeout -> - ?SLOG(error, #{ - msg => "http_connector_pmap_failed", - reason => timeout + ?SLOG(info, #{ + msg => "health_check_failed", + reason => timeout, + connector => PoolName }), {error, timeout} end. From 57074015c6f1a1983193ffe382e7dff29386427b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 17:10:07 -0300 Subject: [PATCH 07/32] feat(ds): allow customizing the data directory The storage expectations for the RocksDB DB may be different from our usual data directory. Also, it may consume a lot more storage than other data. This allows customizing the data directory for the builtin DS storage backend. Note: if the cluster was already initialized using a directory path, changing that config will have no effect. This path is currently persisted in mnesia and used when reopening the DB. --- apps/emqx/src/emqx_persistent_message.erl | 8 ++++- apps/emqx/src/emqx_schema.erl | 30 +++++++++++++++++++ apps/emqx_conf/src/emqx_conf_schema.erl | 18 ++--------- .../src/emqx_ds_storage_layer.erl | 20 +++++++++---- 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_message.erl b/apps/emqx/src/emqx_persistent_message.erl index 295ddd3dc..d725c9b2c 100644 --- a/apps/emqx/src/emqx_persistent_message.erl +++ b/apps/emqx/src/emqx_persistent_message.erl @@ -61,10 +61,16 @@ force_ds() -> emqx_config:get([session_persistence, force_persistence]). storage_backend(#{ - builtin := #{enable := true, n_shards := NShards, replication_factor := ReplicationFactor} + builtin := #{ + enable := true, + data_dir := DataDir, + n_shards := NShards, + replication_factor := ReplicationFactor + } }) -> #{ backend => builtin, + data_dir => DataDir, storage => {emqx_ds_storage_bitfield_lts, #{}}, n_shards => NShards, replication_factor => ReplicationFactor diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ae22db14f..7cd67089d 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -94,6 +94,7 @@ non_empty_string/1, validations/0, naive_env_interpolation/1, + ensure_unicode_path/2, validate_server_ssl_opts/1, validate_tcp_keepalive/1, parse_tcp_keepalive/1 @@ -1882,6 +1883,18 @@ fields("session_storage_backend_builtin") -> default => true } )}, + {"data_dir", + sc( + string(), + #{ + desc => ?DESC(session_builtin_data_dir), + default => <<"${EMQX_DATA_DIR}">>, + importance => ?IMPORTANCE_LOW, + converter => fun(Path, Opts) -> + naive_env_interpolation(ensure_unicode_path(Path, Opts)) + end + } + )}, {"n_shards", sc( pos_integer(), @@ -3836,3 +3849,20 @@ tags_schema() -> importance => ?IMPORTANCE_LOW } ). + +ensure_unicode_path(undefined, _) -> + undefined; +ensure_unicode_path(Path, #{make_serializable := true}) -> + %% format back to serializable string + unicode:characters_to_binary(Path, utf8); +ensure_unicode_path(Path, Opts) when is_binary(Path) -> + case unicode:characters_to_list(Path, utf8) of + {R, _, _} when R =:= error orelse R =:= incomplete -> + throw({"bad_file_path_string", Path}); + PathStr -> + ensure_unicode_path(PathStr, Opts) + end; +ensure_unicode_path(Path, _) when is_list(Path) -> + Path; +ensure_unicode_path(Path, _) -> + throw({"not_string", Path}). diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 6614b24e2..d88a26c1d 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1430,22 +1430,8 @@ convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10); convert_rotation(Count, _Opts) when is_integer(Count) -> Count; convert_rotation(Count, _Opts) -> throw({"bad_rotation", Count}). -ensure_unicode_path(undefined, _) -> - undefined; -ensure_unicode_path(Path, #{make_serializable := true}) -> - %% format back to serializable string - unicode:characters_to_binary(Path, utf8); -ensure_unicode_path(Path, Opts) when is_binary(Path) -> - case unicode:characters_to_list(Path, utf8) of - {R, _, _} when R =:= error orelse R =:= incomplete -> - throw({"bad_file_path_string", Path}); - PathStr -> - ensure_unicode_path(PathStr, Opts) - end; -ensure_unicode_path(Path, _) when is_list(Path) -> - Path; -ensure_unicode_path(Path, _) -> - throw({"not_string", Path}). +ensure_unicode_path(Path, Opts) -> + emqx_schema:ensure_unicode_path(Path, Opts). log_level() -> hoconsc:enum([debug, info, notice, warning, error, critical, alert, emergency, all]). diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl index ab64005b6..d44235924 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl @@ -34,7 +34,7 @@ -export([start_link/2, init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). %% internal exports: --export([db_dir/1]). +-export([db_dir/2]). -export_type([ gen_id/0, @@ -168,7 +168,13 @@ open_shard(Shard, Options) -> -spec drop_shard(shard_id()) -> ok. drop_shard(Shard) -> catch emqx_ds_storage_layer_sup:stop_shard(Shard), - ok = rocksdb:destroy(db_dir(Shard), []). + case persistent_term:get({?MODULE, Shard, data_dir}, undefined) of + undefined -> + ok; + BaseDir -> + ok = rocksdb:destroy(db_dir(BaseDir, Shard), []), + persistent_term:erase({?MODULE, Shard, base_dir}) + end. -spec store_batch(shard_id(), [emqx_types:message()], emqx_ds:message_store_opts()) -> emqx_ds:store_batch_result(). @@ -424,7 +430,8 @@ rocksdb_open(Shard, Options) -> {create_missing_column_families, true} | maps:get(db_options, Options, []) ], - DBDir = db_dir(Shard), + DataDir = maps:get(data_dir, Options, emqx:data_dir()), + DBDir = db_dir(DataDir, Shard), _ = filelib:ensure_dir(DBDir), ExistingCFs = case rocksdb:list_column_families(DBDir, DBOptions) of @@ -440,15 +447,16 @@ rocksdb_open(Shard, Options) -> ], case rocksdb:open(DBDir, DBOptions, ColumnFamilies) of {ok, DBHandle, [_CFDefault | CFRefs]} -> + persistent_term:put({?MODULE, Shard, data_dir}, DataDir), {CFNames, _} = lists:unzip(ExistingCFs), {ok, DBHandle, lists:zip(CFNames, CFRefs)}; Error -> Error end. --spec db_dir(shard_id()) -> file:filename(). -db_dir({DB, ShardId}) -> - filename:join([emqx:data_dir(), atom_to_list(DB), binary_to_list(ShardId)]). +-spec db_dir(file:filename(), shard_id()) -> file:filename(). +db_dir(BaseDir, {DB, ShardId}) -> + filename:join([BaseDir, atom_to_list(DB), binary_to_list(ShardId)]). -spec update_last_until(Schema, emqx_ds:time()) -> Schema when Schema :: shard_schema() | shard(). update_last_until(Schema, Until) -> From 29d767bd625a2430da85ac2ee3dd6c8d18fe0ece Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 16 Jan 2024 14:39:41 +0800 Subject: [PATCH 08/32] ci: add env vars to run cassandra tests locally --- .../docker-compose-toxiproxy.yaml | 4 ++ .ci/docker-compose-file/toxiproxy.json | 12 ++++++ .../test/emqx_bridge_cassandra_SUITE.erl | 8 ++++ .../emqx_bridge_cassandra_connector_SUITE.erl | 40 +++++++++++-------- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index d648d9d78..568d9129c 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -39,6 +39,10 @@ services: - 19042:9042 # Cassandra TLS - 19142:9142 + # Cassandra No Auth + - 19043:9043 + # Cassandra TLS No Auth + - 19143:9143 # S3 - 19000:19000 # S3 TLS diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index c58474039..103bae924 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -96,6 +96,18 @@ "upstream": "cassandra:9142", "enabled": true }, + { + "name": "cassa_no_auth_tcp", + "listen": "0.0.0.0:9043", + "upstream": "cassandra_noauth:9042", + "enabled": true + }, + { + "name": "cassa_no_auth_tls", + "listen": "0.0.0.0:9143", + "upstream": "cassandra_noauth:9142", + "enabled": true + }, { "name": "sqlserver", "listen": "0.0.0.0:1433", diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 9df219296..e0e3900b0 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -11,6 +11,14 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +%% To run this test locally: +%% ./scripts/ct/run.sh --app apps/emqx_bridge_cassandra --only-up +%% PROFILE=emqx-enterprise PROXY_HOST=localhost CASSA_TLS_HOST=localhost \ +%% CASSA_TLS_PORT=19142 CASSA_TCP_HOST=localhost CASSA_TCP_NO_AUTH_HOST=localhost \ +%% CASSA_TCP_PORT=19042 CASSA_TCP_NO_AUTH_PORT=19043 \ +%% ./rebar3 ct --name 'test@127.0.0.1' -v --suite \ +%% apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE + % SQL definitions -define(SQL_BRIDGE, "insert into mqtt_msg_test(topic, payload, arrived) " diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index de306e3f0..245110de6 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -14,20 +14,20 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). +%% To run this test locally: +%% ./scripts/ct/run.sh --app apps/emqx_bridge_cassandra --only-up +%% PROFILE=emqx-enterprise PROXY_HOST=localhost CASSA_TLS_HOST=localhost \ +%% CASSA_TLS_PORT=9142 CASSA_TCP_HOST=localhost CASSA_TCP_NO_AUTH_HOST=localhost \ +%% CASSA_TCP_PORT=19042 CASSA_TCP_NO_AUTH_PORT=19043 \ +%% ./rebar3 ct --name 'test@127.0.0.1' -v --suite \ +%% apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE + %% Cassandra servers are defined at `.ci/docker-compose-file/docker-compose-cassandra.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). -define(CASSANDRA_HOST_NOAUTH, "cassandra_noauth"). -define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_connector). -%% This test SUITE requires a running cassandra instance. If you don't want to -%% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script -%% you can create a cassandra instance with the following command (execute it -%% from root of the EMQX directory.). You also need to set ?CASSANDRA_HOST and -%% ?CASSANDRA_PORT to appropriate values. -%% -%% sudo docker run --rm -d --name cassandra --network host cassandra:3.11.14 - %% Cassandra default username & password once enable `authenticator: PasswordAuthenticator` %% in cassandra config -define(CASSA_USERNAME, <<"cassandra">>). @@ -45,14 +45,14 @@ groups() -> {noauth, [t_lifecycle]} ]. -cassandra_servers(CassandraHost) -> +cassandra_servers(CassandraHost, CassandraPort) -> lists:map( fun(#{hostname := Host, port := Port}) -> {Host, Port} end, emqx_schema:parse_servers( - iolist_to_binary([CassandraHost, ":", erlang:integer_to_list(?CASSANDRA_DEFAULT_PORT)]), - #{default_port => ?CASSANDRA_DEFAULT_PORT} + iolist_to_binary([CassandraHost, ":", erlang:integer_to_list(CassandraPort)]), + #{default_port => CassandraPort} ) ). @@ -63,25 +63,30 @@ init_per_suite(Config) -> Config. init_per_group(Group, Config) -> - {CassandraHost, AuthOpts} = + {CassandraHost, CassandraPort, AuthOpts} = case Group of auth -> - {?CASSANDRA_HOST, [{username, ?CASSA_USERNAME}, {password, ?CASSA_PASSWORD}]}; + TcpHost = os:getenv("CASSA_TCP_HOST", "toxiproxy"), + TcpPort = list_to_integer(os:getenv("CASSA_TCP_PORT", "9042")), + {TcpHost, TcpPort, [{username, ?CASSA_USERNAME}, {password, ?CASSA_PASSWORD}]}; noauth -> - {?CASSANDRA_HOST_NOAUTH, []} + TcpHost = os:getenv("CASSA_TCP_NO_AUTH_HOST", "toxiproxy"), + TcpPort = list_to_integer(os:getenv("CASSA_TCP_NO_AUTH_PORT", "9043")), + {TcpHost, TcpPort, []} end, - case emqx_common_test_helpers:is_tcp_server_available(CassandraHost, ?CASSANDRA_DEFAULT_PORT) of + case emqx_common_test_helpers:is_tcp_server_available(CassandraHost, CassandraPort) of true -> %% keyspace `mqtt` must be created in advance {ok, Conn} = ecql:connect([ - {nodes, cassandra_servers(CassandraHost)}, + {nodes, cassandra_servers(CassandraHost, CassandraPort)}, {keyspace, "mqtt"} | AuthOpts ]), ecql:close(Conn), [ {cassa_host, CassandraHost}, + {cassa_port, CassandraPort}, {cassa_auth_opts, AuthOpts} | Config ]; @@ -212,6 +217,7 @@ create_local_resource(ResourceId, CheckedConfig) -> cassandra_config(Config) -> Host = ?config(cassa_host, Config), + Port = ?config(cassa_port, Config), AuthOpts = maps:from_list(?config(cassa_auth_opts, Config)), CassConfig = AuthOpts#{ @@ -223,7 +229,7 @@ cassandra_config(Config) -> "~s:~b", [ Host, - ?CASSANDRA_DEFAULT_PORT + Port ] ) ) From b32c0fb0d8dc210d12e7a4d02a7f4cf798b93dc5 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 19 Jan 2024 18:46:35 +0800 Subject: [PATCH 09/32] refactor: split cassandra bridges to actions and connectors --- apps/emqx_bridge/src/emqx_action_info.erl | 1 + .../src/emqx_bridge_cassandra.app.src | 4 +- .../src/emqx_bridge_cassandra.erl | 108 ++++++- .../src/emqx_bridge_cassandra_action_info.erl | 62 ++++ .../src/emqx_bridge_cassandra_connector.erl | 286 ++++++++---------- .../test/emqx_bridge_cassandra_SUITE.erl | 21 +- .../emqx_bridge_cassandra_connector_SUITE.erl | 4 - .../src/schema/emqx_connector_ee_schema.erl | 12 + .../src/schema/emqx_connector_schema.erl | 2 + rel/i18n/emqx_bridge_cassandra.hocon | 10 + .../emqx_bridge_cassandra_connector.hocon | 6 + 11 files changed, 332 insertions(+), 184 deletions(-) create mode 100644 apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index d80050191..3754677a8 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -92,6 +92,7 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_matrix_action_info, emqx_bridge_mongodb_action_info, emqx_bridge_influxdb_action_info, + emqx_bridge_cassandra_action_info, emqx_bridge_mysql_action_info, emqx_bridge_pgsql_action_info, emqx_bridge_syskeeper_action_info, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 97be100d2..aa8290b98 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.2.0"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,7 @@ emqx_resource, ecql ]}, - {env, []}, + {env, [{emqx_action_info_modules, [emqx_bridge_cassandra_action_info]}]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 2724b7c09..83268cab5 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -12,11 +12,17 @@ %% schema examples -export([ - conn_bridge_examples/1, values/2, fields/2 ]). +%% Examples +-export([ + bridge_v2_examples/1, + conn_bridge_examples/1, + connector_examples/1 +]). + %% schema -export([ namespace/0, @@ -26,10 +32,13 @@ ]). -define(DEFAULT_CQL, << - "insert into mqtt_msg(topic, msgid, sender, qos, payload, arrived, retain) " - "values (${topic}, ${id}, ${clientid}, ${qos}, ${payload}, ${timestamp}, ${flags.retain})" + "insert into mqtt_msg(msgid, topic, qos, payload, arrived) " + "values (${id}, ${topic}, ${qos}, ${payload}, ${timestamp})" >>). +-define(CONNECTOR_TYPE, cassandra). +-define(ACTION_TYPE, cassandra). + %%-------------------------------------------------------------------- %% schema examples @@ -43,6 +52,41 @@ conn_bridge_examples(Method) -> } ]. +bridge_v2_examples(Method) -> + ParamsExample = #{ + parameters => #{ + cql => ?DEFAULT_CQL + } + }, + [ + #{ + <<"cassandra">> => #{ + summary => <<"Cassandra Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, cassandra, cassandra, ParamsExample + ) + } + } + ]. + +connector_examples(Method) -> + [ + #{ + <<"cassandra">> => #{ + summary => <<"Cassandra Connector">>, + value => emqx_connector_schema:connector_values( + Method, cassandra, #{ + servers => <<"127.0.0.1:9042">>, + keyspace => <<"mqtt">>, + username => <<"root">>, + password => <<"******">>, + pool_size => 8 + } + ) + } + } + ]. + %% no difference in get/post/put method values(_Method, Type) -> #{ @@ -73,14 +117,47 @@ namespace() -> "bridge_cassa". roots() -> []. +fields("config_connector") -> + emqx_connector_schema:common_fields() ++ + emqx_bridge_cassandra_connector:fields("connector") ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts); +fields(action) -> + {cassandra, + mk( + hoconsc:map(name, ref(?MODULE, cassandra_action)), + #{desc => <<"Cassandra Action Config">>, required => false} + )}; +fields(cassandra_action) -> + emqx_bridge_v2_schema:make_producer_action_schema( + mk(ref(?MODULE, action_parameters), #{ + required => true, desc => ?DESC(action_parameters) + }) + ); +fields(action_parameters) -> + [ + cql_field() + ]; +fields(connector_resource_opts) -> + emqx_connector_schema:resource_opts_fields(); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + Fields = + emqx_bridge_cassandra_connector:fields("connector") ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts), + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, Fields); +fields(Field) when + Field == "get_bridge_v2"; + Field == "post_bridge_v2"; + Field == "put_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION_TYPE, fields(cassandra_action)); fields("config") -> [ + cql_field(), {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {cql, - mk( - binary(), - #{desc => ?DESC("cql_template"), default => ?DEFAULT_CQL, format => <<"sql">>} - )}, {local_topic, mk( binary(), @@ -99,8 +176,23 @@ fields("get") -> fields("post", Type) -> [type_field(Type), name_field() | fields("config")]. +cql_field() -> + {cql, + mk( + binary(), + #{desc => ?DESC("cql_template"), default => ?DEFAULT_CQL, format => <<"sql">>} + )}. + desc("config") -> ?DESC("desc_config"); +desc(cassandra_action) -> + ?DESC(cassandra_action); +desc(action_parameters) -> + ?DESC(action_parameters); +desc("config_connector") -> + ?DESC("desc_config"); +desc(connector_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for Cassandra using `", string:to_upper(Method), "` method."]; desc(_) -> diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl new file mode 100644 index 000000000..14db7cf50 --- /dev/null +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_action_info.erl @@ -0,0 +1,62 @@ +-module(emqx_bridge_cassandra_action_info). + +-behaviour(emqx_action_info). + +-export([ + bridge_v1_config_to_action_config/2, + bridge_v1_config_to_connector_config/1, + connector_action_config_to_bridge_v1_config/2, + bridge_v1_type_name/0, + action_type_name/0, + connector_type_name/0, + schema_module/0 +]). + +-import(emqx_utils_conv, [bin/1]). + +-define(SCHEMA_MODULE, emqx_bridge_cassandra). + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + ActionTopLevelKeys = schema_keys(cassandra_action), + ActionParametersKeys = schema_keys(action_parameters), + ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys, + ActionConfig = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_bridge_v2_schema:project_to_actions_resource_opts/1, + ActionConfig#{<<"connector">> => ConnectorName} + ). + +bridge_v1_config_to_connector_config(BridgeV1Config) -> + ActionTopLevelKeys = schema_keys(cassandra_action), + ActionParametersKeys = schema_keys(action_parameters), + ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys, + ConnectorTopLevelKeys = schema_keys("config_connector"), + ConnectorKeys = maps:keys(BridgeV1Config) -- (ActionKeys -- ConnectorTopLevelKeys), + ConnConfig0 = maps:with(ConnectorKeys, BridgeV1Config), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_connector_schema:project_to_connector_resource_opts/1, + ConnConfig0 + ). + +connector_action_config_to_bridge_v1_config(ConnectorRawConf, ActionRawConf) -> + RawConf = emqx_action_info:connector_action_config_to_bridge_v1_config( + ConnectorRawConf, ActionRawConf + ), + maps:without([<<"cassandra_type">>], RawConf). + +bridge_v1_type_name() -> cassandra. + +action_type_name() -> cassandra. + +connector_type_name() -> cassandra. + +schema_module() -> ?SCHEMA_MODULE. + +make_config_map(PickKeys, IndentKeys, Config) -> + Conf0 = maps:with(PickKeys, Config), + emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0). + +schema_keys(Name) -> + [bin(Key) || Key <- proplists:get_keys(?SCHEMA_MODULE:fields(Name))]. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index c6bc7098c..3db71c9e0 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -14,13 +14,17 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% schema --export([roots/0, fields/1]). +-export([roots/0, fields/1, desc/1]). %% callbacks of behaviour emqx_resource -export([ callback_mode/0, on_start/2, on_stop/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channel_status/3, + on_get_channels/1, on_query/3, on_query_async/4, on_batch_query/3, @@ -28,6 +32,8 @@ on_get_status/2 ]). +-export([transform_bridge_v1_config_to_connector_config/1]). + %% callbacks of ecpool -export([ connect/1, @@ -39,16 +45,10 @@ -export([do_get_status/1]). --type prepares() :: #{atom() => binary()}. --type params_tokens() :: #{atom() => list()}. - -type state() :: #{ pool_name := binary(), - prepare_cql := prepares(), - params_tokens := params_tokens(), - %% returned by ecql:prepare/2 - prepare_statement := binary() + channels := #{} }. -define(DEFAULT_SERVER_OPTION, #{default_port => ?CASSANDRA_DEFAULT_PORT}). @@ -62,7 +62,9 @@ roots() -> fields(config) -> cassandra_db_fields() ++ emqx_connector_schema_lib:ssl_fields() ++ - emqx_connector_schema_lib:prepare_statement_fields(). + emqx_connector_schema_lib:prepare_statement_fields(); +fields("connector") -> + cassandra_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). cassandra_db_fields() -> [ @@ -83,6 +85,11 @@ keyspace(desc) -> ?DESC("keyspace"); keyspace(required) -> true; keyspace(_) -> undefined. +desc(config) -> + ?DESC("config"); +desc("connector") -> + ?DESC("connector"). + %%-------------------------------------------------------------------- %% callbacks for emqx_resource @@ -130,10 +137,9 @@ on_start( false -> [] end, - State = parse_prepare_cql(Config), case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; + {ok, #{pool_name => InstId, channels => #{}}}; {error, Reason} -> ?tp( cassandra_connector_start_failed, @@ -149,23 +155,49 @@ on_stop(InstId, _State) -> }), emqx_resource_pool:stop(InstId). +on_add_channel(_InstId, #{channels := Channs} = OldState, ChannId, ChannConf0) -> + #{parameters := #{cql := CQL}} = ChannConf0, + {PrepareCQL, ParamsTokens} = emqx_placeholder:preproc_sql(CQL, '?'), + ParsedCql = #{ + prepare_key => short_prepare_key(ChannId), + prepare_cql => PrepareCQL, + params_tokens => ParamsTokens + }, + NewChanns = Channs#{ChannId => #{parsed_cql => ParsedCql, prepare_result => not_prepared}}, + {ok, OldState#{channels => NewChanns}}. + +on_remove_channel(_InstanceId, #{channels := Channels} = State, ChannId) -> + NewState = State#{channels => maps:remove(ChannId, Channels)}, + {ok, NewState}. + +on_get_channel_status(InstanceId, ChannId, #{channels := Channels, pool_name := PoolName} = State) -> + case on_get_status(InstanceId, State) of + connected -> + #{parsed_cql := ParsedCql} = maps:get(ChannId, Channels), + case prepare_cql_to_cassandra(ParsedCql, PoolName) of + {ok, _} -> connected; + {error, Reason} -> {connecting, Reason} + end; + _ -> + connecting + end. + +on_get_channels(InstanceId) -> + emqx_bridge_v2:get_channels_for_connector(InstanceId). + -type request() :: % emqx_bridge.erl - {send_message, Params :: map()} + {ChannId :: binary(), Params :: map()} % common query - | {query, SQL :: binary()} - | {query, SQL :: binary(), Params :: map()}. + | {query, CQL :: binary()} + | {query, CQL :: binary(), Params :: map()}. -spec on_query( emqx_resource:resource_id(), request(), state() ) -> ok | {ok, ecql:cql_result()} | {error, {recoverable_error | unrecoverable_error, term()}}. -on_query( - InstId, - Request, - State -) -> +on_query(InstId, Request, State) -> do_single_query(InstId, Request, sync, State). -spec on_query_async( @@ -174,21 +206,11 @@ on_query( {function(), list()}, state() ) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. -on_query_async( - InstId, - Request, - Callback, - State -) -> +on_query_async(InstId, Request, Callback, State) -> do_single_query(InstId, Request, {async, Callback}, State). -do_single_query( - InstId, - Request, - Async, - #{pool_name := PoolName} = State -) -> - {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), +do_single_query(InstId, Request, Async, #{pool_name := PoolName} = State) -> + {Type, PreparedKeyOrCQL, Params} = parse_request_to_cql(Request), ?tp( debug, cassandra_connector_received_cql_query, @@ -196,12 +218,12 @@ do_single_query( connector => InstId, type => Type, params => Params, - prepared_key_or_cql => PreparedKeyOrSQL, + prepared_key_or_cql => PreparedKeyOrCQL, state => State } ), - {PreparedKeyOrSQL1, Data} = proc_cql_params(Type, PreparedKeyOrSQL, Params, State), - Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrSQL1, Data), + {PreparedKeyOrCQL1, Data} = proc_cql_params(Type, PreparedKeyOrCQL, Params, State), + Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrCQL1, Data), handle_result(Res). -spec on_batch_query( @@ -209,11 +231,7 @@ do_single_query( [request()], state() ) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. -on_batch_query( - InstId, - Requests, - State -) -> +on_batch_query(InstId, Requests, State) -> do_batch_query(InstId, Requests, sync, State). -spec on_batch_query_async( @@ -222,25 +240,15 @@ on_batch_query( {function(), list()}, state() ) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. -on_batch_query_async( - InstId, - Requests, - Callback, - State -) -> +on_batch_query_async(InstId, Requests, Callback, State) -> do_batch_query(InstId, Requests, {async, Callback}, State). -do_batch_query( - InstId, - Requests, - Async, - #{pool_name := PoolName} = State -) -> +do_batch_query(InstId, Requests, Async, #{pool_name := PoolName} = State) -> CQLs = lists:map( fun(Request) -> - {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), - proc_cql_params(Type, PreparedKeyOrSQL, Params, State) + {Type, PreparedKeyOrCQL, Params} = parse_request_to_cql(Request), + proc_cql_params(Type, PreparedKeyOrCQL, Params, State) end, Requests ), @@ -256,26 +264,24 @@ do_batch_query( Res = exec_cql_batch_query(InstId, PoolName, Async, CQLs), handle_result(Res). -parse_request_to_cql({send_message, Params}) -> - {prepared_query, _Key = send_message, Params}; -parse_request_to_cql({query, SQL}) -> - parse_request_to_cql({query, SQL, #{}}); -parse_request_to_cql({query, SQL, Params}) -> - {query, SQL, Params}. +parse_request_to_cql({query, CQL}) -> + {query, CQL, #{}}; +parse_request_to_cql({query, CQL, Params}) -> + {query, CQL, Params}; +parse_request_to_cql({ChannId, Params}) -> + {prepared_query, ChannId, Params}. -proc_cql_params( - prepared_query, - PreparedKey0, - Params, - #{prepare_statement := Prepares, params_tokens := ParamsTokens} -) -> - %% assert - _PreparedKey = maps:get(PreparedKey0, Prepares), - Tokens = maps:get(PreparedKey0, ParamsTokens), - {PreparedKey0, assign_type_for_params(emqx_placeholder:proc_sql(Tokens, Params))}; -proc_cql_params(query, SQL, Params, _State) -> - {SQL1, Tokens} = emqx_placeholder:preproc_sql(SQL, '?'), - {SQL1, assign_type_for_params(emqx_placeholder:proc_sql(Tokens, Params))}. +proc_cql_params(prepared_query, ChannId, Params, #{channels := Channs}) -> + #{ + parsed_cql := #{ + prepare_key := PrepareKey, + params_tokens := ParamsTokens + } + } = maps:get(ChannId, Channs), + {PrepareKey, assign_type_for_params(emqx_placeholder:proc_sql(ParamsTokens, Params))}; +proc_cql_params(query, CQL, Params, _State) -> + {CQL1, Tokens} = emqx_placeholder:preproc_sql(CQL, '?'), + {CQL1, assign_type_for_params(emqx_placeholder:proc_sql(Tokens, Params))}. exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when Type == query; Type == prepared_query @@ -314,38 +320,15 @@ exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> exec(PoolName, Query) -> ecpool:pick_and_do(PoolName, Query, no_handover). -on_get_status(_InstId, #{pool_name := PoolName} = State) -> +on_get_status(_InstId, #{pool_name := PoolName}) -> case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of - true -> - case do_check_prepares(State) of - ok -> - connected; - {ok, NState} -> - %% return new state with prepared statements - {connected, NState}; - false -> - %% do not log error, it is logged in prepare_cql_to_conn - connecting - end; - false -> - connecting + true -> connected; + false -> connecting end. do_get_status(Conn) -> ok == element(1, ecql:query(Conn, "SELECT cluster_name FROM system.local")). -do_check_prepares(#{prepare_cql := Prepares}) when is_map(Prepares) -> - ok; -do_check_prepares(State = #{pool_name := PoolName, prepare_cql := {error, Prepares}}) -> - %% retry to prepare - case prepare_cql(Prepares, PoolName) of - {ok, Sts} -> - %% remove the error - {ok, State#{prepare_cql => Prepares, prepare_statement := Sts}}; - _Error -> - false - end. - %%-------------------------------------------------------------------- %% callbacks query @@ -394,88 +377,50 @@ conn_opts([Opt | Opts], Acc) -> %%-------------------------------------------------------------------- %% prepare - -%% XXX: hardcode -%% note: the `cql` param is passed by emqx_bridge_cassandra -parse_prepare_cql(#{cql := SQL}) -> - parse_prepare_cql([{send_message, SQL}], #{}, #{}); -parse_prepare_cql(_) -> - #{prepare_cql => #{}, params_tokens => #{}}. - -parse_prepare_cql([{Key, H} | T], Prepares, Tokens) -> - {PrepareSQL, ParamsTokens} = emqx_placeholder:preproc_sql(H, '?'), - parse_prepare_cql( - T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} - ); -parse_prepare_cql([], Prepares, Tokens) -> - #{ - prepare_cql => Prepares, - params_tokens => Tokens - }. - -init_prepare(State = #{prepare_cql := Prepares, pool_name := PoolName}) -> - case maps:size(Prepares) of - 0 -> - State; - _ -> - case prepare_cql(Prepares, PoolName) of - {ok, Sts} -> - State#{prepare_statement := Sts}; - Error -> - ?tp( - error, - cassandra_prepare_cql_failed, - #{prepares => Prepares, reason => Error} - ), - %% mark the prepare_cql as failed - State#{prepare_cql => {error, Prepares}} - end - end. - -prepare_cql(Prepares, PoolName) when is_map(Prepares) -> - prepare_cql(maps:to_list(Prepares), PoolName); -prepare_cql(Prepares, PoolName) -> - case do_prepare_cql(Prepares, PoolName) of - {ok, _Sts} = Ok -> +prepare_cql_to_cassandra(ParsedCql, PoolName) -> + case prepare_cql_to_cassandra(ecpool:workers(PoolName), ParsedCql, #{}) of + {ok, Statement} -> %% prepare for reconnect - ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_cql_to_conn, [Prepares]}), - Ok; + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_cql_to_conn, [ParsedCql]}), + {ok, Statement}; Error -> + ?tp( + error, + cassandra_prepare_cql_failed, + #{parsed_cql => ParsedCql, reason => Error} + ), Error end. -do_prepare_cql(Prepares, PoolName) -> - do_prepare_cql(ecpool:workers(PoolName), Prepares, #{}). - -do_prepare_cql([{_Name, Worker} | T], Prepares, _LastSts) -> +prepare_cql_to_cassandra([{_Name, Worker} | T], ParsedCql, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), - case prepare_cql_to_conn(Conn, Prepares) of - {ok, Sts} -> - do_prepare_cql(T, Prepares, Sts); + case prepare_cql_to_conn(Conn, ParsedCql) of + {ok, Statement} -> + prepare_cql_to_cassandra(T, ParsedCql, Statement); Error -> Error end; -do_prepare_cql([], _Prepares, LastSts) -> +prepare_cql_to_cassandra([], _ParsedCql, LastSts) -> {ok, LastSts}. -prepare_cql_to_conn(Conn, Prepares) -> - prepare_cql_to_conn(Conn, Prepares, #{}). - -prepare_cql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; -prepare_cql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> - ?SLOG(info, #{msg => "cassandra_prepare_cql", name => Key, prepare_cql => SQL}), - case ecql:prepare(Conn, Key, SQL) of +prepare_cql_to_conn(Conn, #{prepare_key := PrepareKey, prepare_cql := PrepareCQL}) when + is_pid(Conn) +-> + ?SLOG(info, #{ + msg => "cassandra_prepare_cql", prepare_key => PrepareKey, prepare_cql => PrepareCQL + }), + case ecql:prepare(Conn, PrepareKey, PrepareCQL) of {ok, Statement} -> - prepare_cql_to_conn(Conn, PrepareList, Statements#{Key => Statement}); - {error, Error} = Other -> + {ok, Statement}; + {error, Reason} = Error -> ?SLOG(error, #{ msg => "cassandra_prepare_cql_failed", worker_pid => Conn, - name => Key, - prepare_cql => SQL, - error => Error + name => PrepareKey, + prepare_cql => PrepareCQL, + reason => Reason }), - Other + Error end. handle_result({error, disconnected}) -> @@ -487,6 +432,9 @@ handle_result({error, Error}) -> handle_result(Res) -> Res. +transform_bridge_v1_config_to_connector_config(_) -> + ok. + %%-------------------------------------------------------------------- %% utils @@ -513,3 +461,11 @@ maybe_assign_type(V) when is_integer(V) -> maybe_assign_type(V) when is_float(V) -> {double, V}; maybe_assign_type(V) -> V. + +short_prepare_key(Str) when is_binary(Str) -> + true = size(Str) > 0, + Sha = crypto:hash(sha, Str), + %% TODO: change to binary:encode_hex(X, lowercase) when OTP version is always > 25 + Hex = string:lowercase(binary:encode_hex(Sha)), + <> = Hex, + binary_to_atom(<<"cassa_prepare_key:", UniqueEnough/binary>>). diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index e0e3900b0..77aec7d99 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -301,17 +301,28 @@ send_message(Config, Payload) -> query_resource(Config, Request) -> Name = ?config(cassa_name, Config), BridgeType = ?config(cassa_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + BridgeV2Id = emqx_bridge_v2:id(BridgeType, Name), + ConnectorResId = emqx_connector_resource:resource_id( + cassandra, <<"connector_emqx_bridge_cassandra_SUITE">> + ), + emqx_resource:query(BridgeV2Id, Request, #{ + timeout => 1_000, connector_resource_id => ConnectorResId + }). query_resource_async(Config, Request) -> Name = ?config(cassa_name, Config), BridgeType = ?config(cassa_bridge_type, Config), Ref = alias([reply]), AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - Return = emqx_resource:query(ResourceID, Request, #{ - timeout => 500, async_reply_fun => {AsyncReplyFun, []} + BridgeV2Id = emqx_bridge_v2:id(BridgeType, Name), + ConnectorResId = emqx_connector_resource:resource_id( + cassandra, <<"connector_emqx_bridge_cassandra_SUITE">> + ), + Return = emqx_resource:query(BridgeV2Id, Request, #{ + timeout => 500, + async_reply_fun => {AsyncReplyFun, []}, + connector_resource_id => ConnectorResId, + query_mode => async }), {Return, Ref}. diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 245110de6..50d82397a 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -22,10 +22,6 @@ %% ./rebar3 ct --name 'test@127.0.0.1' -v --suite \ %% apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE -%% Cassandra servers are defined at `.ci/docker-compose-file/docker-compose-cassandra.yaml` -%% You can change it to `127.0.0.1`, if you run this SUITE locally --define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_HOST_NOAUTH, "cassandra_noauth"). -define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_connector). %% Cassandra default username & password once enable `authenticator: PasswordAuthenticator` diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 655892d88..2b50252e8 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -36,6 +36,8 @@ resource_type(mongodb) -> emqx_bridge_mongodb_connector; resource_type(influxdb) -> emqx_bridge_influxdb_connector; +resource_type(cassandra) -> + emqx_bridge_cassandra_connector; resource_type(mysql) -> emqx_bridge_mysql_connector; resource_type(pgsql) -> @@ -130,6 +132,14 @@ connector_structs() -> required => false } )}, + {cassandra, + mk( + hoconsc:map(name, ref(emqx_bridge_cassandra, "config_connector")), + #{ + desc => <<"Cassandra Connector Config">>, + required => false + } + )}, {mysql, mk( hoconsc:map(name, ref(emqx_bridge_mysql, "config_connector")), @@ -205,6 +215,7 @@ schema_modules() -> emqx_bridge_matrix, emqx_bridge_mongodb, emqx_bridge_influxdb, + emqx_bridge_cassandra, emqx_bridge_mysql, emqx_bridge_syskeeper_connector, emqx_bridge_syskeeper_proxy, @@ -234,6 +245,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_matrix, <<"matrix">>, Method ++ "_connector"), api_ref(emqx_bridge_mongodb, <<"mongodb">>, Method ++ "_connector"), api_ref(emqx_bridge_influxdb, <<"influxdb">>, Method ++ "_connector"), + api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method ++ "_connector"), api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"), api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method), diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 615b89230..f4bbb5459 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -137,6 +137,8 @@ connector_type_to_bridge_types(mongodb) -> [mongodb, mongodb_rs, mongodb_sharded, mongodb_single]; connector_type_to_bridge_types(influxdb) -> [influxdb, influxdb_api_v1, influxdb_api_v2]; +connector_type_to_bridge_types(cassandra) -> + [cassandra]; connector_type_to_bridge_types(mysql) -> [mysql]; connector_type_to_bridge_types(mqtt) -> diff --git a/rel/i18n/emqx_bridge_cassandra.hocon b/rel/i18n/emqx_bridge_cassandra.hocon index a96315340..29eb35de5 100644 --- a/rel/i18n/emqx_bridge_cassandra.hocon +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -1,5 +1,15 @@ emqx_bridge_cassandra { +action_parameters.desc: +"""Action specific configs.""" +action_parameters.label: +"""Action""" + +cassandra_action.desc: +"""Action configs.""" +cassandra_action.label: +"""Action""" + config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_bridge_cassandra_connector.hocon b/rel/i18n/emqx_bridge_cassandra_connector.hocon index b149cce8a..40e1c0e22 100644 --- a/rel/i18n/emqx_bridge_cassandra_connector.hocon +++ b/rel/i18n/emqx_bridge_cassandra_connector.hocon @@ -1,5 +1,11 @@ emqx_bridge_cassandra_connector { +config.desc: +"""Cassandra connection config""" + +config.label: +"""Connection config""" + keyspace.desc: """Keyspace name to connect to.""" From 0e1043f80cc397618d15bf6a24af30b3a3ed31d4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Sun, 21 Jan 2024 21:00:38 +0800 Subject: [PATCH 10/32] ci: update generated connector name --- .../test/emqx_bridge_cassandra_SUITE.erl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 77aec7d99..09deaa699 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -302,9 +302,7 @@ query_resource(Config, Request) -> Name = ?config(cassa_name, Config), BridgeType = ?config(cassa_bridge_type, Config), BridgeV2Id = emqx_bridge_v2:id(BridgeType, Name), - ConnectorResId = emqx_connector_resource:resource_id( - cassandra, <<"connector_emqx_bridge_cassandra_SUITE">> - ), + ConnectorResId = emqx_connector_resource:resource_id(BridgeType, Name), emqx_resource:query(BridgeV2Id, Request, #{ timeout => 1_000, connector_resource_id => ConnectorResId }). @@ -315,9 +313,7 @@ query_resource_async(Config, Request) -> Ref = alias([reply]), AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, BridgeV2Id = emqx_bridge_v2:id(BridgeType, Name), - ConnectorResId = emqx_connector_resource:resource_id( - cassandra, <<"connector_emqx_bridge_cassandra_SUITE">> - ), + ConnectorResId = emqx_connector_resource:resource_id(BridgeType, Name), Return = emqx_resource:query(BridgeV2Id, Request, #{ timeout => 500, async_reply_fun => {AsyncReplyFun, []}, From 8118297ddc80faac701ff9bb22d0bcb21977163c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 03:44:31 +0000 Subject: [PATCH 11/32] chore(deps): bump the actions-package-macos group Bumps the actions-package-macos group in /.github/actions/package-macos with 1 update: [actions/cache](https://github.com/actions/cache). Updates `actions/cache` from 3.3.3 to 4.0.0 - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/e12d46a63a90f2fae62d114769bbf2a179198b5c...13aacd865c20de90d75de3b17ebe84f7a17d57d2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions-package-macos ... Signed-off-by: dependabot[bot] --- .github/actions/package-macos/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 64d179b46..1553576b2 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -51,7 +51,7 @@ runs: echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT ;; esac - - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3 + - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 id: cache if: steps.prepare.outputs.SELF_HOSTED != 'true' with: From 837b19cb1e6023a90195db0052f00aee8ace3e4f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 22 Jan 2024 15:45:00 +0800 Subject: [PATCH 12/32] chore: update change logs for cassandra bridge_v2 --- changes/ee/feat-12330.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-12330.en.md diff --git a/changes/ee/feat-12330.en.md b/changes/ee/feat-12330.en.md new file mode 100644 index 000000000..963098659 --- /dev/null +++ b/changes/ee/feat-12330.en.md @@ -0,0 +1 @@ +The bridges for Cassandra have been split so they are available via the connectors and actions APIs. They are still backwards compatible with the old bridge API. From e337e1dc408369cc8486f2ee60ee3b2701f9c26a Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 22 Jan 2024 20:45:10 +0800 Subject: [PATCH 13/32] feat(opents): improve the OpentsDB bridge to v2 style --- apps/emqx_bridge/src/emqx_action_info.erl | 3 +- .../src/emqx_bridge_opents.erl | 128 ++++- .../src/emqx_bridge_opents_action_info.erl | 71 +++ .../src/emqx_bridge_opents_connector.erl | 194 ++++++- .../test/emqx_bridge_opents_SUITE.erl | 484 +++++++----------- .../src/schema/emqx_connector_ee_schema.erl | 18 +- .../src/schema/emqx_connector_schema.erl | 4 +- rel/i18n/emqx_bridge_opents.hocon | 31 ++ rel/i18n/emqx_bridge_opents_connector.hocon | 6 + 9 files changed, 607 insertions(+), 332 deletions(-) create mode 100644 apps/emqx_bridge_opents/src/emqx_bridge_opents_action_info.erl diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index d80050191..4f6228998 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -98,7 +98,8 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_timescale_action_info, emqx_bridge_redis_action_info, emqx_bridge_iotdb_action_info, - emqx_bridge_es_action_info + emqx_bridge_es_action_info, + emqx_bridge_opents_action_info ]. -else. hard_coded_action_info_modules_ee() -> diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl index cfb12453d..7e490576f 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_bridge_opents). @@ -7,10 +7,12 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). --import(hoconsc, [mk/2, enum/1, ref/2]). +-import(hoconsc, [mk/2, enum/1, ref/2, array/1]). -export([ - conn_bridge_examples/1 + conn_bridge_examples/1, + bridge_v2_examples/1, + default_data_template/0 ]). -export([ @@ -20,8 +22,11 @@ desc/1 ]). +-define(CONNECTOR_TYPE, opents). +-define(ACTION_TYPE, ?CONNECTOR_TYPE). + %% ------------------------------------------------------------------------------------------------- -%% api +%% v1 examples conn_bridge_examples(Method) -> [ #{ @@ -34,7 +39,7 @@ conn_bridge_examples(Method) -> values(_Method) -> #{ - enable => true, + enabledb => true, type => opents, name => <<"foo">>, server => <<"http://127.0.0.1:4242">>, @@ -50,7 +55,37 @@ values(_Method) -> }. %% ------------------------------------------------------------------------------------------------- -%% Hocon Schema Definitions +%% v2 examples +bridge_v2_examples(Method) -> + [ + #{ + <<"opents">> => #{ + summary => <<"OpenTSDB Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, ?ACTION_TYPE, ?CONNECTOR_TYPE, action_values() + ) + } + } + ]. + +action_values() -> + #{ + parameters => #{ + data => default_data_template() + } + }. + +default_data_template() -> + [ + #{ + metric => <<"${metric}">>, + tags => <<"${tags}">>, + value => <<"${value}">> + } + ]. + +%% ------------------------------------------------------------------------------------------------- +%% V1 Schema Definitions namespace() -> "bridge_opents". roots() -> []. @@ -65,10 +100,89 @@ fields("post") -> fields("put") -> fields("config"); fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). + emqx_bridge_schema:status_fields() ++ fields("post"); +%% ------------------------------------------------------------------------------------------------- +%% V2 Schema Definitions + +fields(action) -> + {opents, + mk( + hoconsc:map(name, ref(?MODULE, action_config)), + #{ + desc => <<"OpenTSDB Action Config">>, + required => false + } + )}; +fields(action_config) -> + emqx_bridge_v2_schema:make_producer_action_schema( + mk( + ref(?MODULE, action_parameters), + #{ + required => true, desc => ?DESC("action_parameters") + } + ) + ); +fields(action_parameters) -> + [ + {data, + mk( + array(ref(?MODULE, action_parameters_data)), + #{ + desc => ?DESC("action_parameters_data"), + default => <<"[]">> + } + )} + ]; +fields(action_parameters_data) -> + [ + {timestamp, + mk( + binary(), + #{ + desc => ?DESC("config_parameters_timestamp"), + required => false + } + )}, + {metric, + mk( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_metric") + } + )}, + {tags, + mk( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_tags") + } + )}, + {value, + mk( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_value") + } + )} + ]; +fields("post_bridge_v2") -> + emqx_bridge_schema:type_and_name_fields(enum([opents])) ++ fields(action_config); +fields("put_bridge_v2") -> + fields(action_config); +fields("get_bridge_v2") -> + emqx_bridge_schema:status_fields() ++ fields("post_bridge_v2"). desc("config") -> ?DESC("desc_config"); +desc(action_config) -> + ?DESC("desc_config"); +desc(action_parameters) -> + ?DESC("action_parameters"); +desc(action_parameters_data) -> + ?DESC("action_parameters_data"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for OpenTSDB using `", string:to_upper(Method), "` method."]; desc(_) -> diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_action_info.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_action_info.erl new file mode 100644 index 000000000..4c4c9568c --- /dev/null +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_action_info.erl @@ -0,0 +1,71 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_opents_action_info). + +-behaviour(emqx_action_info). + +-elvis([{elvis_style, invalid_dynamic_call, disable}]). + +%% behaviour callbacks +-export([ + action_type_name/0, + bridge_v1_config_to_action_config/2, + bridge_v1_config_to_connector_config/1, + bridge_v1_type_name/0, + connector_action_config_to_bridge_v1_config/2, + connector_type_name/0, + schema_module/0 +]). + +-import(emqx_utils_conv, [bin/1]). + +-define(ACTION_TYPE, opents). +-define(SCHEMA_MODULE, emqx_bridge_opents). + +action_type_name() -> ?ACTION_TYPE. +bridge_v1_type_name() -> ?ACTION_TYPE. +connector_type_name() -> ?ACTION_TYPE. + +schema_module() -> ?SCHEMA_MODULE. + +connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> + MergedConfig = + emqx_utils_maps:deep_merge( + maps:without( + [<<"description">>, <<"local_topic">>, <<"connector">>, <<"data">>], + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) + ), + ConnectorConfig + ), + BridgeV1Keys = schema_keys("config"), + maps:with(BridgeV1Keys, MergedConfig). + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + ActionTopLevelKeys = schema_keys(action_config), + ActionParametersKeys = schema_keys(action_parameters), + ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys, + ActionConfig = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_bridge_v2_schema:project_to_actions_resource_opts/1, + ActionConfig#{<<"connector">> => ConnectorName} + ). + +bridge_v1_config_to_connector_config(BridgeV1Config) -> + ConnectorKeys = schema_keys(emqx_bridge_opents_connector, "config_connector"), + emqx_utils_maps:update_if_present( + <<"resource_opts">>, + fun emqx_connector_schema:project_to_connector_resource_opts/1, + maps:with(ConnectorKeys, BridgeV1Config) + ). + +make_config_map(PickKeys, IndentKeys, Config) -> + Conf0 = maps:with(PickKeys, Config#{<<"data">> => []}), + emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0). + +schema_keys(Name) -> + schema_keys(?SCHEMA_MODULE, Name). + +schema_keys(Mod, Name) -> + [bin(Key) || Key <- proplists:get_keys(Mod:fields(Name))]. diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 9271abe15..6af1e2f55 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_bridge_opents_connector). @@ -12,7 +12,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1, desc/1]). %% `emqx_resource' API -export([ @@ -21,15 +21,25 @@ on_stop/2, on_query/3, on_batch_query/3, - on_get_status/2 + on_get_status/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channels/1, + on_get_channel_status/3 ]). +-export([connector_examples/1]). + -export([connect/1]). -import(hoconsc, [mk/2, enum/1, ref/2]). +-define(CONNECTOR_TYPE, opents). + +namespace() -> "opents_connector". + %%===================================================================== -%% Hocon schema +%% V1 Hocon schema roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. @@ -40,8 +50,56 @@ fields(config) -> {summary, mk(boolean(), #{default => true, desc => ?DESC("summary")})}, {details, mk(boolean(), #{default => false, desc => ?DESC("details")})}, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} + ]; +%%===================================================================== +%% V2 Hocon schema + +fields("config_connector") -> + emqx_connector_schema:common_fields() ++ + proplists_without([auto_reconnect], fields(config)); +fields("post") -> + emqx_connector_schema:type_and_name_fields(enum([opents])) ++ fields("config_connector"); +fields("put") -> + fields("config_connector"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc(config) -> + ?DESC("desc_config"); +desc("config_connector") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for IoTDB using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +proplists_without(Keys, List) -> + [El || El = {K, _} <- List, not lists:member(K, Keys)]. + +%%===================================================================== +%% V2 examples +connector_examples(Method) -> + [ + #{ + <<"opents">> => + #{ + summary => <<"OpenTSDB Connector">>, + value => emqx_connector_schema:connector_values( + Method, ?CONNECTOR_TYPE, connector_example_values() + ) + } + } ]. +connector_example_values() -> + #{ + name => <<"opents_connector">>, + type => opents, + enable => true, + server => <<"http://localhost:4242/">>, + pool_size => 8 + }. + %%======================================================================================== %% `emqx_resource' API %%======================================================================================== @@ -56,8 +114,7 @@ on_start( server := Server, pool_size := PoolSize, summary := Summary, - details := Details, - resource_opts := #{batch_size := BatchSize} + details := Details } = Config ) -> ?SLOG(info, #{ @@ -70,11 +127,10 @@ on_start( {server, to_str(Server)}, {summary, Summary}, {details, Details}, - {max_batch_size, BatchSize}, {pool_size, PoolSize} ], - State = #{pool_name => InstanceId, server => Server}, + State = #{pool_name => InstanceId, server => Server, channels => #{}}, case opentsdb_connectivity(Server) of ok -> case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of @@ -93,6 +149,7 @@ on_stop(InstanceId, _State) -> msg => "stopping_opents_connector", connector => InstanceId }), + ?tp(opents_bridge_stopped, #{instance_id => InstanceId}), emqx_resource_pool:stop(InstanceId). on_query(InstanceId, Request, State) -> @@ -101,10 +158,14 @@ on_query(InstanceId, Request, State) -> on_batch_query( InstanceId, BatchReq, - State + #{channels := Channels} = State ) -> - Datas = [format_opentsdb_msg(Msg) || {_Key, Msg} <- BatchReq], - do_query(InstanceId, Datas, State). + case try_render_messages(BatchReq, Channels) of + {ok, Datas} -> + do_query(InstanceId, Datas, State); + Error -> + Error + end. on_get_status(_InstanceId, #{server := Server}) -> Result = @@ -117,6 +178,39 @@ on_get_status(_InstanceId, #{server := Server}) -> end, Result. +on_add_channel( + _InstanceId, + #{channels := Channels} = OldState, + ChannelId, + #{ + parameters := #{data := Data} = Parameter + } +) -> + case maps:is_key(ChannelId, Channels) of + true -> + {error, already_exists}; + _ -> + Channel = Parameter#{ + data := preproc_data_template(Data) + }, + Channels2 = Channels#{ChannelId => Channel}, + {ok, OldState#{channels := Channels2}} + end. + +on_remove_channel(_InstanceId, #{channels := Channels} = OldState, ChannelId) -> + {ok, OldState#{channels => maps:remove(ChannelId, Channels)}}. + +on_get_channels(InstanceId) -> + emqx_bridge_v2:get_channels_for_connector(InstanceId). + +on_get_channel_status(InstanceId, ChannelId, #{channels := Channels} = State) -> + case maps:is_key(ChannelId, Channels) of + true -> + on_get_status(InstanceId, State); + _ -> + {error, not_exists} + end. + %%======================================================================================== %% Helper fns %%======================================================================================== @@ -127,6 +221,9 @@ do_query(InstanceId, Query, #{pool_name := PoolName} = State) -> "opents_connector_received", #{connector => InstanceId, query => Query, state => State} ), + + ?tp(opents_bridge_on_query, #{instance_id => InstanceId}), + Result = ecpool:pick_and_do(PoolName, {opentsdb, put, [Query]}, no_handover), case Result of @@ -172,17 +269,66 @@ opentsdb_connectivity(Server) -> end, emqx_connector_lib:http_connectivity(SvrUrl, ?HTTP_CONNECT_TIMEOUT). -format_opentsdb_msg(Msg) -> - maps:with( - [ - timestamp, - metric, - tags, - value, - <<"timestamp">>, - <<"metric">>, - <<"tags">>, - <<"value">> - ], - Msg +try_render_messages([{ChannelId, _} | _] = BatchReq, Channels) -> + case maps:find(ChannelId, Channels) of + {ok, Channel} -> + {ok, + lists:foldl( + fun({_, Message}, Acc) -> + render_channel_message(Message, Channel, Acc) + end, + [], + BatchReq + )}; + _ -> + {error, {unrecoverable_error, {invalid_channel_id, ChannelId}}} + end. + +render_channel_message(Msg, #{data := DataList}, Acc) -> + RawOpts = #{return => rawlist, var_trans => fun(X) -> X end}, + lists:foldl( + fun(#{metric := MetricTk, tags := TagsTk, value := ValueTk} = Data, InAcc) -> + MetricVal = emqx_placeholder:proc_tmpl(MetricTk, Msg), + TagsVal = + case emqx_placeholder:proc_tmpl(TagsTk, Msg, RawOpts) of + [undefined] -> + #{}; + [Any] -> + Any + end, + ValueVal = + case ValueTk of + [_] -> + erlang:hd(emqx_placeholder:proc_tmpl(ValueTk, Msg, RawOpts)); + _ -> + emqx_placeholder:proc_tmpl(ValueTk, Msg) + end, + Base = #{metric => MetricVal, tags => TagsVal, value => ValueVal}, + [ + case maps:get(timestamp, Data, undefined) of + undefined -> + Base; + TimestampTk -> + Base#{timestamp => emqx_placeholder:proc_tmpl(TimestampTk, Msg)} + end + | InAcc + ] + end, + Acc, + DataList + ). + +preproc_data_template([]) -> + preproc_data_template(emqx_bridge_opents:default_data_template()); +preproc_data_template(DataList) -> + lists:map( + fun(Data) -> + maps:map( + fun(_Key, Value) -> + emqx_placeholder:preproc_tmpl(Value) + end, + Data + ) + end, + DataList ). diff --git a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl index 3632ce786..f86ae6986 100644 --- a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl +++ b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_bridge_opents_SUITE). @@ -12,7 +12,8 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). % DB defaults --define(BATCH_SIZE, 10). +-define(BRIDGE_TYPE_BIN, <<"opents">>). +-define(APPS, [opentsdb, emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_opents_SUITE]). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -20,95 +21,34 @@ all() -> [ - {group, with_batch}, - {group, without_batch} + {group, default} ]. groups() -> - TCs = emqx_common_test_helpers:all(?MODULE), + AllTCs = emqx_common_test_helpers:all(?MODULE), [ - {with_batch, TCs}, - {without_batch, TCs} + {default, AllTCs} ]. -init_per_group(with_batch, Config0) -> - Config = [{batch_size, ?BATCH_SIZE} | Config0], - common_init(Config); -init_per_group(without_batch, Config0) -> - Config = [{batch_size, 1} | Config0], - common_init(Config); -init_per_group(_Group, Config) -> - Config. - -end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> - ProxyHost = ?config(proxy_host, Config), - ProxyPort = ?config(proxy_port, Config), - emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - ok; -end_per_group(_Group, _Config) -> - ok. - init_per_suite(Config) -> - Config. + emqx_bridge_v2_testlib:init_per_suite(Config, ?APPS). -end_per_suite(_Config) -> - emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([opentsdb, emqx_bridge, emqx_resource, emqx_conf]), - ok. +end_per_suite(Config) -> + emqx_bridge_v2_testlib:end_per_suite(Config). -init_per_testcase(_Testcase, Config) -> - delete_bridge(Config), - snabbkaffe:start_trace(), - Config. - -end_per_testcase(_Testcase, Config) -> - ProxyHost = ?config(proxy_host, Config), - ProxyPort = ?config(proxy_port, Config), - emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - ok = snabbkaffe:stop(), - delete_bridge(Config), - ok. - -%%------------------------------------------------------------------------------ -%% Helper fns -%%------------------------------------------------------------------------------ - -common_init(ConfigT) -> - Host = os:getenv("OPENTS_HOST", "toxiproxy"), +init_per_group(default, Config0) -> + Host = os:getenv("OPENTS_HOST", "toxiproxy.emqx.net"), Port = list_to_integer(os:getenv("OPENTS_PORT", "4242")), - - Config0 = [ - {opents_host, Host}, - {opents_port, Port}, - {proxy_name, "opents"} - | ConfigT - ], - - BridgeType = proplists:get_value(bridge_type, Config0, <<"opents">>), + ProxyName = "opents", case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of true -> - % Setup toxiproxy - ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), - ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), - emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure enterprise bridge module is loaded - ok = emqx_common_test_helpers:start_apps([ - emqx_conf, emqx_resource, emqx_bridge - ]), - _ = application:ensure_all_started(opentsdb), - _ = emqx_bridge_enterprise:module_info(), - emqx_mgmt_api_test_util:init_suite(), - {Name, OpenTSConf} = opents_config(BridgeType, Config0), - Config = - [ - {opents_config, OpenTSConf}, - {opents_bridge_type, BridgeType}, - {opents_name, Name}, - {proxy_host, ProxyHost}, - {proxy_port, ProxyPort} - | Config0 - ], - Config; + Config = emqx_bridge_v2_testlib:init_per_group(default, ?BRIDGE_TYPE_BIN, Config0), + [ + {bridge_host, Host}, + {bridge_port, Port}, + {proxy_name, ProxyName} + | Config + ]; false -> case os:getenv("IS_CI") of "yes" -> @@ -116,244 +56,152 @@ common_init(ConfigT) -> _ -> {skip, no_opents} end - end. - -opents_config(BridgeType, Config) -> - Port = integer_to_list(?config(opents_port, Config)), - Server = "http://" ++ ?config(opents_host, Config) ++ ":" ++ Port, - Name = atom_to_binary(?MODULE), - BatchSize = ?config(batch_size, Config), - ConfigString = - io_lib:format( - "bridges.~s.~s {\n" - " enable = true\n" - " server = ~p\n" - " resource_opts = {\n" - " request_ttl = 500ms\n" - " batch_size = ~b\n" - " query_mode = sync\n" - " }\n" - "}", - [ - BridgeType, - Name, - Server, - BatchSize - ] - ), - {Name, parse_and_check(ConfigString, BridgeType, Name)}. - -parse_and_check(ConfigString, BridgeType, Name) -> - {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), - hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), - #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + end; +init_per_group(_Group, Config) -> Config. -create_bridge(Config) -> - create_bridge(Config, _Overrides = #{}). +end_per_group(default, Config) -> + emqx_bridge_v2_testlib:end_per_group(Config), + ok; +end_per_group(_Group, _Config) -> + ok. -create_bridge(Config, Overrides) -> - BridgeType = ?config(opents_bridge_type, Config), - Name = ?config(opents_name, Config), - Config0 = ?config(opents_config, Config), - Config1 = emqx_utils_maps:deep_merge(Config0, Overrides), - emqx_bridge:create(BridgeType, Name, Config1). +init_per_testcase(TestCase, Config0) -> + Type = ?config(bridge_type, Config0), + UniqueNum = integer_to_binary(erlang:unique_integer()), + Name = << + (atom_to_binary(TestCase))/binary, UniqueNum/binary + >>, + {_ConfigString, ConnectorConfig} = connector_config(Name, Config0), + {_, ActionConfig} = action_config(Name, Config0), + Config = [ + {connector_type, Type}, + {connector_name, Name}, + {connector_config, ConnectorConfig}, + {bridge_type, Type}, + {bridge_name, Name}, + {bridge_config, ActionConfig} + | Config0 + ], + %% iotdb_reset(Config), + ok = snabbkaffe:start_trace(), + Config. -delete_bridge(Config) -> - BridgeType = ?config(opents_bridge_type, Config), - Name = ?config(opents_name, Config), - emqx_bridge:remove(BridgeType, Name). - -create_bridge_http(Params) -> - Path = emqx_mgmt_api_test_util:api_path(["bridges"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; - Error -> Error - end. - -send_message(Config, Payload) -> - Name = ?config(opents_name, Config), - BridgeType = ?config(opents_bridge_type, Config), - BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), - emqx_bridge:send_message(BridgeID, Payload). - -query_resource(Config, Request) -> - query_resource(Config, Request, 1_000). - -query_resource(Config, Request, Timeout) -> - Name = ?config(opents_name, Config), - BridgeType = ?config(opents_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - emqx_resource:query(ResourceID, Request, #{timeout => Timeout}). +end_per_testcase(TestCase, Config) -> + emqx_bridge_v2_testlib:end_per_testcase(TestCase, Config). %%------------------------------------------------------------------------------ -%% Testcases +%% Helper fns %%------------------------------------------------------------------------------ -t_setup_via_config_and_publish(Config) -> - ?assertMatch( - {ok, _}, - create_bridge(Config) - ), - SentData = make_data(), - ?check_trace( - begin - {_, {ok, #{result := Result}}} = - ?wait_async_action( - send_message(Config, SentData), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - ?assertMatch( - {ok, 200, #{failed := 0, success := 1}}, Result - ), - ok - end, - fun(Trace0) -> - Trace = ?of_kind(opents_connector_query_return, Trace0), - ?assertMatch([#{result := {ok, 200, #{failed := 0, success := 1}}}], Trace), - ok - end - ), - ok. +action_config(Name, Config) -> + Type = ?config(bridge_type, Config), + ConfigString = + io_lib:format( + "actions.~s.~s {\n" + " enable = true\n" + " connector = \"~s\"\n" + " parameters = {\n" + " data = []\n" + " }\n" + "}\n", + [ + Type, + Name, + Name + ] + ), + ct:pal("ActionConfig:~ts~n", [ConfigString]), + {ConfigString, parse_action_and_check(ConfigString, Type, Name)}. -t_setup_via_http_api_and_publish(Config) -> - BridgeType = ?config(opents_bridge_type, Config), - Name = ?config(opents_name, Config), - OpentsConfig0 = ?config(opents_config, Config), - OpentsConfig = OpentsConfig0#{ - <<"name">> => Name, - <<"type">> => BridgeType - }, - ?assertMatch( - {ok, _}, - create_bridge_http(OpentsConfig) - ), - SentData = make_data(), - ?check_trace( - begin - Request = {send_message, SentData}, - Res0 = query_resource(Config, Request, 2_500), - ?assertMatch( - {ok, 200, #{failed := 0, success := 1}}, Res0 - ), - ok - end, - fun(Trace0) -> - Trace = ?of_kind(opents_connector_query_return, Trace0), - ?assertMatch([#{result := {ok, 200, #{failed := 0, success := 1}}}], Trace), - ok - end - ), - ok. +connector_config(Name, Config) -> + Host = ?config(bridge_host, Config), + Port = ?config(bridge_port, Config), + Type = ?config(bridge_type, Config), + ServerURL = opents_server_url(Host, Port), + ConfigString = + io_lib:format( + "connectors.~s.~s {\n" + " enable = true\n" + " server = \"~s\"\n" + "}\n", + [ + Type, + Name, + ServerURL + ] + ), + ct:pal("ConnectorConfig:~ts~n", [ConfigString]), + {ConfigString, parse_connector_and_check(ConfigString, Type, Name)}. -t_get_status(Config) -> - ?assertMatch( - {ok, _}, - create_bridge(Config) - ), +parse_action_and_check(ConfigString, BridgeType, Name) -> + parse_and_check(ConfigString, emqx_bridge_schema, <<"actions">>, BridgeType, Name). - Name = ?config(opents_name, Config), - BridgeType = ?config(opents_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), +parse_connector_and_check(ConfigString, ConnectorType, Name) -> + parse_and_check( + ConfigString, emqx_connector_schema, <<"connectors">>, ConnectorType, Name + ). +%% emqx_utils_maps:safe_atom_key_map(Config). - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), - ok. +parse_and_check(ConfigString, SchemaMod, RootKey, Type0, Name) -> + Type = to_bin(Type0), + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(SchemaMod, RawConf, #{required => false, atom_key => false}), + #{RootKey := #{Type := #{Name := Config}}} = RawConf, + Config. -t_create_disconnected(Config) -> - BridgeType = proplists:get_value(bridge_type, Config, <<"opents">>), - Config1 = lists:keyreplace(opents_port, 1, Config, {opents_port, 61234}), - {_Name, OpenTSConf} = opents_config(BridgeType, Config1), +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8); +to_bin(Atom) when is_atom(Atom) -> + erlang:atom_to_binary(Atom); +to_bin(Bin) when is_binary(Bin) -> + Bin. - Config2 = lists:keyreplace(opents_config, 1, Config1, {opents_config, OpenTSConf}), - ?assertMatch({ok, _}, create_bridge(Config2)), +opents_server_url(Host, Port) -> + iolist_to_binary([ + "http://", + Host, + ":", + integer_to_binary(Port) + ]). - Name = ?config(opents_name, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceID)), - ok. +is_success_check({ok, 200, #{failed := Failed}}) -> + ?assertEqual(0, Failed); +is_success_check(Ret) -> + ?assert(false, Ret). -t_write_failure(Config) -> - ProxyName = ?config(proxy_name, Config), - ProxyPort = ?config(proxy_port, Config), - ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge(Config), - SentData = make_data(), - emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - {_, {ok, #{result := Result}}} = - ?wait_async_action( - send_message(Config, SentData), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - ?assertMatch({error, _}, Result), - ok - end), - ok. +is_error_check(Result) -> + ?assertMatch({error, {400, #{failed := 1}}}, Result). -t_write_timeout(Config) -> - ProxyName = ?config(proxy_name, Config), - ProxyPort = ?config(proxy_port, Config), - ProxyHost = ?config(proxy_host, Config), - {ok, _} = create_bridge( - Config, - #{ - <<"resource_opts">> => #{ - <<"request_ttl">> => <<"500ms">>, - <<"resume_interval">> => <<"100ms">>, - <<"health_check_interval">> => <<"100ms">> +opentds_query(Config, Metric) -> + Path = <<"/api/query">>, + Opts = #{return_all => true}, + Body = #{ + start => <<"1h-ago">>, + queries => [ + #{ + aggregator => <<"last">>, + metric => Metric, + tags => #{ + host => <<"*">> + } } - } - ), - SentData = make_data(), - emqx_common_test_helpers:with_failure( - timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData}) - ) - end - ), - ok. + ], + showTSUID => false, + showQuery => false, + delete => false + }, + opentsdb_request(Config, Path, Body, Opts). -t_missing_data(Config) -> - ?assertMatch( - {ok, _}, - create_bridge(Config) - ), - {_, {ok, #{result := Result}}} = - ?wait_async_action( - send_message(Config, #{}), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - ?assertMatch( - {error, {400, #{failed := 1, success := 0}}}, - Result - ), - ok. +opentsdb_request(Config, Path, Body) -> + opentsdb_request(Config, Path, Body, #{}). -t_bad_data(Config) -> - ?assertMatch( - {ok, _}, - create_bridge(Config) - ), - Data = maps:without([metric], make_data()), - {_, {ok, #{result := Result}}} = - ?wait_async_action( - send_message(Config, Data), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - - ?assertMatch( - {error, {400, #{failed := 1, success := 0}}}, Result - ), - ok. - -make_data() -> - make_data(<<"cpu">>, 12). +opentsdb_request(Config, Path, Body, Opts) -> + Host = ?config(bridge_host, Config), + Port = ?config(bridge_port, Config), + ServerURL = opents_server_url(Host, Port), + URL = <>, + emqx_mgmt_api_test_util:request_api(post, URL, [], [], Body, Opts). make_data(Metric, Value) -> #{ @@ -363,3 +211,45 @@ make_data(Metric, Value) -> }, value => Value }. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_query_simple(Config) -> + Metric = <<"t_query_simple">>, + Value = 12, + MakeMessageFun = fun() -> make_data(Metric, Value) end, + ok = emqx_bridge_v2_testlib:t_sync_query( + Config, MakeMessageFun, fun is_success_check/1, opents_bridge_on_query + ), + {ok, {{_, 200, _}, _, IoTDBResult}} = opentds_query(Config, Metric), + QResult = emqx_utils_json:decode(IoTDBResult), + ?assertMatch( + [ + #{ + <<"metric">> := Metric, + <<"dps">> := _ + } + ], + QResult + ), + [#{<<"dps">> := Dps}] = QResult, + ?assertMatch([Value | _], maps:values(Dps)). + +t_create_via_http(Config) -> + emqx_bridge_v2_testlib:t_create_via_http(Config). + +t_start_stop(Config) -> + emqx_bridge_v2_testlib:t_start_stop(Config, opents_bridge_stopped). + +t_on_get_status(Config) -> + emqx_bridge_v2_testlib:t_on_get_status(Config, #{failure_status => connecting}). + +t_query_invalid_data(Config) -> + Metric = <<"t_query_invalid_data">>, + Value = 12, + MakeMessageFun = fun() -> maps:remove(value, make_data(Metric, Value)) end, + ok = emqx_bridge_v2_testlib:t_sync_query( + Config, MakeMessageFun, fun is_error_check/1, opents_bridge_on_query + ). diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 655892d88..90c1ae1ce 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -52,6 +52,8 @@ resource_type(iotdb) -> emqx_bridge_iotdb_connector; resource_type(elasticsearch) -> emqx_bridge_es_connector; +resource_type(opents) -> + emqx_bridge_opents_connector; resource_type(Type) -> error({unknown_connector_type, Type}). @@ -66,6 +68,8 @@ connector_impl_module(iotdb) -> emqx_bridge_iotdb_connector; connector_impl_module(elasticsearch) -> emqx_bridge_es_connector; +connector_impl_module(opents) -> + emqx_bridge_opents_connector; connector_impl_module(_ConnectorType) -> undefined. @@ -193,6 +197,14 @@ connector_structs() -> desc => <<"ElasticSearch Connector Config">>, required => false } + )}, + {opents, + mk( + hoconsc:map(name, ref(emqx_bridge_opents_connector, "config_connector")), + #{ + desc => <<"OpenTSDB Connector Config">>, + required => false + } )} ]. @@ -212,7 +224,8 @@ schema_modules() -> emqx_postgresql_connector_schema, emqx_bridge_redis_schema, emqx_bridge_iotdb_connector, - emqx_bridge_es_connector + emqx_bridge_es_connector, + emqx_bridge_opents_connector ]. api_schemas(Method) -> @@ -241,7 +254,8 @@ api_schemas(Method) -> api_ref(emqx_postgresql_connector_schema, <<"pgsql">>, Method ++ "_connector"), api_ref(emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector"), api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), - api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method) + api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method), + api_ref(emqx_bridge_opents_connector, <<"opents">>, Method) ]. api_ref(Module, Type, Method) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 615b89230..1829e04e6 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -154,7 +154,9 @@ connector_type_to_bridge_types(timescale) -> connector_type_to_bridge_types(iotdb) -> [iotdb]; connector_type_to_bridge_types(elasticsearch) -> - [elasticsearch]. + [elasticsearch]; +connector_type_to_bridge_types(opents) -> + [opents]. actions_config_name(action) -> <<"actions">>; actions_config_name(source) -> <<"sources">>. diff --git a/rel/i18n/emqx_bridge_opents.hocon b/rel/i18n/emqx_bridge_opents.hocon index ff44a9e18..5f1c4b0af 100644 --- a/rel/i18n/emqx_bridge_opents.hocon +++ b/rel/i18n/emqx_bridge_opents.hocon @@ -23,4 +23,35 @@ emqx_bridge_opents { desc_name.label: "Bridge Name" + +action_parameters_data.desc: +"""OpenTSDB action parameter data""" + +action_parameters_data.label: +"""Parameter Data""" + +config_parameters_timestamp.desc: +"""Timestamp. Placeholders in format of ${var} is supported""" + +config_parameters_timestamp.label: +"""Timestamp""" + +config_parameters_metric.metric: +"""Metric. Placeholders in format of ${var} is supported""" + +config_parameters_metric.metric: +"""Metric""" + +config_parameters_tags.desc: +"""Data Type, Placeholders in format of ${var} is supported""" + +config_parameters_tags.label: +"""Tags""" + +config_parameters_value.desc: +"""Value. Placeholders in format of ${var} is supported""" + +config_parameters_value.label: +"""Value""" + } diff --git a/rel/i18n/emqx_bridge_opents_connector.hocon b/rel/i18n/emqx_bridge_opents_connector.hocon index 5c39d1e0e..a54c240a0 100644 --- a/rel/i18n/emqx_bridge_opents_connector.hocon +++ b/rel/i18n/emqx_bridge_opents_connector.hocon @@ -17,4 +17,10 @@ emqx_bridge_opents_connector { details.label: "Details" + +desc_config.desc: +"""Configuration for OpenTSDB Connector.""" + +desc_config.label: +"""OpenTSDB Connector Configuration""" } From 3f7b913e884294e5d13b8c16a78135060a13880c Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 18 Jan 2024 22:25:58 +0800 Subject: [PATCH 14/32] chore(opents): bump version && update changes --- apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src | 2 +- .../src/emqx_bridge_opents_connector.erl | 7 ++++++- changes/ee/feat-12353.en.md | 1 + rel/i18n/emqx_bridge_opents.hocon | 10 ++++++++-- 4 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 changes/ee/feat-12353.en.md diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 5e3b2f585..2469acaa8 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 6af1e2f55..e3fe9d6b4 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -56,7 +56,10 @@ fields(config) -> fields("config_connector") -> emqx_connector_schema:common_fields() ++ - proplists_without([auto_reconnect], fields(config)); + proplists_without([auto_reconnect], fields(config)) ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts); +fields(connector_resource_opts) -> + emqx_connector_schema:resource_opts_fields(); fields("post") -> emqx_connector_schema:type_and_name_fields(enum([opents])) ++ fields("config_connector"); fields("put") -> @@ -66,6 +69,8 @@ fields("get") -> desc(config) -> ?DESC("desc_config"); +desc(connector_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); desc("config_connector") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> diff --git a/changes/ee/feat-12353.en.md b/changes/ee/feat-12353.en.md new file mode 100644 index 000000000..2d34e1211 --- /dev/null +++ b/changes/ee/feat-12353.en.md @@ -0,0 +1 @@ +The bridges for OpentsDB have been split so it is available via the connectors and actions APIs. They are still backwards compatible with the old bridge API. diff --git a/rel/i18n/emqx_bridge_opents.hocon b/rel/i18n/emqx_bridge_opents.hocon index 5f1c4b0af..f5d2ade85 100644 --- a/rel/i18n/emqx_bridge_opents.hocon +++ b/rel/i18n/emqx_bridge_opents.hocon @@ -24,6 +24,12 @@ emqx_bridge_opents { desc_name.label: "Bridge Name" +action_parameters.desc: +"""OpenTSDB action parameters""" + +action_parameters.label: +"""Parameters""" + action_parameters_data.desc: """OpenTSDB action parameter data""" @@ -36,10 +42,10 @@ config_parameters_timestamp.desc: config_parameters_timestamp.label: """Timestamp""" -config_parameters_metric.metric: +config_parameters_metric.desc: """Metric. Placeholders in format of ${var} is supported""" -config_parameters_metric.metric: +config_parameters_metric.label: """Metric""" config_parameters_tags.desc: From 75b08b525b33ef50b9006a54631252ac753c4856 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jan 2024 16:52:59 -0300 Subject: [PATCH 15/32] feat(ds): add `list_generations` and `drop_generation` APIs --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_durable_storage/src/emqx_ds.erl | 57 +++- .../src/emqx_ds_replication_layer.erl | 45 ++- .../src/emqx_ds_storage_bitfield_lts.erl | 16 + .../src/emqx_ds_storage_layer.erl | 293 ++++++++++++++---- .../src/emqx_ds_storage_reference.erl | 8 +- .../src/proto/emqx_ds_proto_v3.erl | 147 +++++++++ .../test/emqx_ds_SUITE.erl | 250 ++++++++++++++- 8 files changed, 755 insertions(+), 62 deletions(-) create mode 100644 apps/emqx_durable_storage/src/proto/emqx_ds_proto_v3.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 2777aec53..2b25cf4be 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -21,6 +21,7 @@ {emqx_delayed,3}. {emqx_ds,1}. {emqx_ds,2}. +{emqx_ds,3}. {emqx_eviction_agent,1}. {emqx_eviction_agent,2}. {emqx_exhook,1}. diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index d679f7097..434169520 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -22,7 +22,14 @@ -module(emqx_ds). %% Management API: --export([open_db/2, update_db_config/2, add_generation/1, drop_db/1]). +-export([ + open_db/2, + update_db_config/2, + add_generation/1, + list_generations_with_lifetimes/1, + drop_generation/2, + drop_db/1 +]). %% Message storage API: -export([store_batch/2, store_batch/3]). @@ -52,7 +59,10 @@ get_iterator_result/1, ds_specific_stream/0, - ds_specific_iterator/0 + ds_specific_iterator/0, + ds_specific_generation_rank/0, + generation_rank/0, + generation_info/0 ]). %%================================================================================ @@ -80,6 +90,8 @@ -type ds_specific_stream() :: term(). +-type ds_specific_generation_rank() :: term(). + -type message_key() :: binary(). -type store_batch_result() :: ok | {error, _}. @@ -114,6 +126,17 @@ -type get_iterator_result(Iterator) :: {ok, Iterator} | undefined. +%% An opaque term identifying a generation. Each implementation will possibly add +%% information to this term to match its inner structure (e.g.: by embedding the shard id, +%% in the case of `emqx_ds_replication_layer'). +-opaque generation_rank() :: ds_specific_generation_rank(). + +-type generation_info() :: #{ + created_at := time(), + since := time(), + until := time() | undefined +}. + -define(persistent_term(DB), {emqx_ds_db_backend, DB}). -define(module(DB), (persistent_term:get(?persistent_term(DB)))). @@ -128,6 +151,11 @@ -callback update_db_config(db(), create_db_opts()) -> ok | {error, _}. +-callback list_generations_with_lifetimes(db()) -> + #{generation_rank() => generation_info()}. + +-callback drop_generation(db(), generation_rank()) -> ok | {error, _}. + -callback drop_db(db()) -> ok | {error, _}. -callback store_batch(db(), [emqx_types:message()], message_store_opts()) -> store_batch_result(). @@ -142,6 +170,11 @@ -callback next(db(), Iterator, pos_integer()) -> next_result(Iterator). +-optional_callbacks([ + list_generations_with_lifetimes/1, + drop_generation/2 +]). + %%================================================================================ %% API funcions %%================================================================================ @@ -166,6 +199,26 @@ add_generation(DB) -> update_db_config(DB, Opts) -> ?module(DB):update_db_config(DB, Opts). +-spec list_generations_with_lifetimes(db()) -> #{generation_rank() => generation_info()}. +list_generations_with_lifetimes(DB) -> + Mod = ?module(DB), + case erlang:function_exported(Mod, list_generations_with_lifetimes, 1) of + true -> + Mod:list_generations_with_lifetimes(DB); + false -> + #{} + end. + +-spec drop_generation(db(), generation_rank()) -> ok | {error, _}. +drop_generation(DB, GenId) -> + Mod = ?module(DB), + case erlang:function_exported(Mod, drop_generation, 2) of + true -> + Mod:drop_generation(DB, GenId); + false -> + {error, not_implemented} + end. + %% @doc TODO: currently if one or a few shards are down, they won't be %% deleted. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl index 68d9459ee..387587570 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl @@ -25,6 +25,8 @@ open_db/2, add_generation/1, update_db_config/2, + list_generations_with_lifetimes/1, + drop_generation/2, drop_db/1, store_batch/3, get_streams/3, @@ -41,7 +43,9 @@ do_make_iterator_v1/5, do_update_iterator_v2/4, do_next_v1/4, - do_add_generation_v2/1 + do_add_generation_v2/1, + do_list_generations_with_lifetimes_v3/2, + do_drop_generation_v3/3 ]). -export_type([shard_id/0, builtin_db_opts/0, stream/0, iterator/0, message_id/0, batch/0]). @@ -104,6 +108,8 @@ ?batch_messages := [emqx_types:message()] }. +-type generation_rank() :: {shard_id(), term()}. + %%================================================================================ %% API functions %%================================================================================ @@ -135,6 +141,32 @@ add_generation(DB) -> update_db_config(DB, CreateOpts) -> emqx_ds_replication_layer_meta:update_db_config(DB, CreateOpts). +-spec list_generations_with_lifetimes(emqx_ds:db()) -> + #{generation_rank() => emqx_ds:generation_info()}. +list_generations_with_lifetimes(DB) -> + Shards = list_shards(DB), + lists:foldl( + fun(Shard, GensAcc) -> + Node = node_of_shard(DB, Shard), + maps:fold( + fun(GenId, Data, AccInner) -> + AccInner#{{Shard, GenId} => Data} + end, + GensAcc, + emqx_ds_proto_v3:list_generations_with_lifetimes(Node, DB, Shard) + ) + end, + #{}, + Shards + ). + +-spec drop_generation(emqx_ds:db(), generation_rank()) -> ok | {error, _}. +drop_generation(DB, {Shard, GenId}) -> + %% TODO: drop generation in all nodes in the replica set, not only in the leader, + %% after we have proper replication in place. + Node = node_of_shard(DB, Shard), + emqx_ds_proto_v3:drop_generation(Node, DB, Shard, GenId). + -spec drop_db(emqx_ds:db()) -> ok | {error, _}. drop_db(DB) -> Nodes = list_nodes(), @@ -301,7 +333,6 @@ do_next_v1(DB, Shard, Iter, BatchSize) -> -spec do_add_generation_v2(emqx_ds:db()) -> ok | {error, _}. do_add_generation_v2(DB) -> MyShards = emqx_ds_replication_layer_meta:my_owned_shards(DB), - lists:foreach( fun(ShardId) -> emqx_ds_storage_layer:add_generation({DB, ShardId}) @@ -309,6 +340,16 @@ do_add_generation_v2(DB) -> MyShards ). +-spec do_list_generations_with_lifetimes_v3(emqx_ds:db(), shard_id()) -> + #{emqx_ds:ds_specific_generation_rank() => emqx_ds:generation_info()}. +do_list_generations_with_lifetimes_v3(DB, ShardId) -> + emqx_ds_storage_layer:list_generations_with_lifetimes({DB, ShardId}). + +-spec do_drop_generation_v3(emqx_ds:db(), shard_id(), emqx_ds_storage_layer:gen_id()) -> + ok | {error, _}. +do_drop_generation_v3(DB, ShardId, GenId) -> + emqx_ds_storage_layer:drop_generation({DB, ShardId}, GenId). + %%================================================================================ %% Internal functions %%================================================================================ diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl index 4c59a5f62..27d41e6c6 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl @@ -27,6 +27,7 @@ -export([ create/4, open/5, + drop/5, store_batch/4, get_streams/4, make_iterator/5, @@ -199,6 +200,21 @@ open(_Shard, DBHandle, GenId, CFRefs, Schema) -> ts_offset = TSOffsetBits }. +-spec drop( + emqx_ds_storage_layer:shard_id(), + rocksdb:db_handle(), + emqx_ds_storage_layer:gen_id(), + emqx_ds_storage_layer:cf_refs(), + s() +) -> + ok. +drop(_Shard, DBHandle, GenId, CFRefs, #s{}) -> + {_, DataCF} = lists:keyfind(data_cf(GenId), 1, CFRefs), + {_, TrieCF} = lists:keyfind(trie_cf(GenId), 1, CFRefs), + ok = rocksdb:drop_column_family(DBHandle, DataCF), + ok = rocksdb:drop_column_family(DBHandle, TrieCF), + ok. + -spec store_batch( emqx_ds_storage_layer:shard_id(), s(), [emqx_types:message()], emqx_ds:message_store_opts() ) -> diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl index ab64005b6..ed161d290 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl @@ -27,7 +27,9 @@ update_iterator/3, next/3, update_config/2, - add_generation/1 + add_generation/1, + list_generations_with_lifetimes/1, + drop_generation/2 ]). %% gen_server @@ -44,7 +46,8 @@ iterator/0, shard_id/0, options/0, - prototype/0 + prototype/0, + post_creation_context/0 ]). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -95,11 +98,18 @@ %%%% Generation: +-define(GEN_KEY(GEN_ID), {generation, GEN_ID}). + -type generation(Data) :: #{ %% Module that handles data for the generation: module := module(), %% Module-specific data defined at generation creation time: data := Data, + %% Column families used by this generation + cf_refs := cf_refs(), + %% Time at which this was created. Might differ from `since', in particular for the + %% first generation. + created_at := emqx_ds:time(), %% When should this generation become active? %% This generation should only contain messages timestamped no earlier than that. %% The very first generation will have `since` equal 0. @@ -121,7 +131,7 @@ %% This data is used to create new generation: prototype := prototype(), %% Generations: - {generation, gen_id()} => GenData + ?GEN_KEY(gen_id()) => GenData }. %% Shard schema (persistent): @@ -132,6 +142,18 @@ -type options() :: map(). +-type post_creation_context() :: + #{ + shard_id := emqx_ds_storage_layer:shard_id(), + db := rocksdb:db_handle(), + new_gen_id := emqx_ds_storage_layer:gen_id(), + old_gen_id := emqx_ds_storage_layer:gen_id(), + new_cf_refs := cf_refs(), + old_cf_refs := cf_refs(), + new_gen_runtime_data := _NewData, + old_gen_runtime_data := _OldData + }. + %%================================================================================ %% Generation callbacks %%================================================================================ @@ -145,6 +167,9 @@ -callback open(shard_id(), rocksdb:db_handle(), gen_id(), cf_refs(), _Schema) -> _Data. +-callback drop(shard_id(), rocksdb:db_handle(), gen_id(), cf_refs(), _RuntimeData) -> + ok | {error, _Reason}. + -callback store_batch(shard_id(), _Data, [emqx_types:message()], emqx_ds:message_store_opts()) -> emqx_ds:store_batch_result(). @@ -157,10 +182,17 @@ -callback next(shard_id(), _Data, Iter, pos_integer()) -> {ok, Iter, [emqx_types:message()]} | {error, _}. +-callback post_creation_actions(post_creation_context()) -> _Data. + +-optional_callbacks([post_creation_actions/1]). + %%================================================================================ %% API for the replication layer %%================================================================================ +-record(call_list_generations_with_lifetimes, {}). +-record(call_drop_generation, {gen_id :: gen_id()}). + -spec open_shard(shard_id(), options()) -> ok. open_shard(Shard, Options) -> emqx_ds_storage_layer_sup:ensure_shard(Shard, Options). @@ -182,18 +214,25 @@ store_batch(Shard, Messages, Options) -> [{integer(), stream()}]. get_streams(Shard, TopicFilter, StartTime) -> Gens = generations_since(Shard, StartTime), + ?tp(get_streams_all_gens, #{gens => Gens}), lists:flatmap( fun(GenId) -> - #{module := Mod, data := GenData} = generation_get(Shard, GenId), - Streams = Mod:get_streams(Shard, GenData, TopicFilter, StartTime), - [ - {GenId, #{ - ?tag => ?STREAM, - ?generation => GenId, - ?enc => Stream - }} - || Stream <- Streams - ] + ?tp(get_streams_get_gen, #{gen_id => GenId}), + case generation_get_safe(Shard, GenId) of + {ok, #{module := Mod, data := GenData}} -> + Streams = Mod:get_streams(Shard, GenData, TopicFilter, StartTime), + [ + {GenId, #{ + ?tag => ?STREAM, + ?generation => GenId, + ?enc => Stream + }} + || Stream <- Streams + ]; + {error, not_found} -> + %% race condition: generation was dropped before getting its streams? + [] + end end, Gens ). @@ -203,16 +242,20 @@ get_streams(Shard, TopicFilter, StartTime) -> make_iterator( Shard, #{?tag := ?STREAM, ?generation := GenId, ?enc := Stream}, TopicFilter, StartTime ) -> - #{module := Mod, data := GenData} = generation_get(Shard, GenId), - case Mod:make_iterator(Shard, GenData, Stream, TopicFilter, StartTime) of - {ok, Iter} -> - {ok, #{ - ?tag => ?IT, - ?generation => GenId, - ?enc => Iter - }}; - {error, _} = Err -> - Err + case generation_get_safe(Shard, GenId) of + {ok, #{module := Mod, data := GenData}} -> + case Mod:make_iterator(Shard, GenData, Stream, TopicFilter, StartTime) of + {ok, Iter} -> + {ok, #{ + ?tag => ?IT, + ?generation => GenId, + ?enc => Iter + }}; + {error, _} = Err -> + Err + end; + {error, not_found} -> + {error, end_of_stream} end. -spec update_iterator( @@ -224,33 +267,42 @@ update_iterator( #{?tag := ?IT, ?generation := GenId, ?enc := OldIter}, DSKey ) -> - #{module := Mod, data := GenData} = generation_get(Shard, GenId), - case Mod:update_iterator(Shard, GenData, OldIter, DSKey) of - {ok, Iter} -> - {ok, #{ - ?tag => ?IT, - ?generation => GenId, - ?enc => Iter - }}; - {error, _} = Err -> - Err + case generation_get_safe(Shard, GenId) of + {ok, #{module := Mod, data := GenData}} -> + case Mod:update_iterator(Shard, GenData, OldIter, DSKey) of + {ok, Iter} -> + {ok, #{ + ?tag => ?IT, + ?generation => GenId, + ?enc => Iter + }}; + {error, _} = Err -> + Err + end; + {error, not_found} -> + {error, end_of_stream} end. -spec next(shard_id(), iterator(), pos_integer()) -> emqx_ds:next_result(iterator()). next(Shard, Iter = #{?tag := ?IT, ?generation := GenId, ?enc := GenIter0}, BatchSize) -> - #{module := Mod, data := GenData} = generation_get(Shard, GenId), - Current = generation_current(Shard), - case Mod:next(Shard, GenData, GenIter0, BatchSize) of - {ok, _GenIter, []} when GenId < Current -> - %% This is a past generation. Storage layer won't write - %% any more messages here. The iterator reached the end: - %% the stream has been fully replayed. - {ok, end_of_stream}; - {ok, GenIter, Batch} -> - {ok, Iter#{?enc := GenIter}, Batch}; - Error = {error, _} -> - Error + case generation_get_safe(Shard, GenId) of + {ok, #{module := Mod, data := GenData}} -> + Current = generation_current(Shard), + case Mod:next(Shard, GenData, GenIter0, BatchSize) of + {ok, _GenIter, []} when GenId < Current -> + %% This is a past generation. Storage layer won't write + %% any more messages here. The iterator reached the end: + %% the stream has been fully replayed. + {ok, end_of_stream}; + {ok, GenIter, Batch} -> + {ok, Iter#{?enc := GenIter}, Batch}; + Error = {error, _} -> + Error + end; + {error, not_found} -> + %% generation was possibly dropped by GC + {ok, end_of_stream} end. -spec update_config(shard_id(), emqx_ds:create_db_opts()) -> ok. @@ -261,6 +313,21 @@ update_config(ShardId, Options) -> add_generation(ShardId) -> gen_server:call(?REF(ShardId), add_generation, infinity). +-spec list_generations_with_lifetimes(shard_id()) -> + #{ + gen_id() => #{ + created_at := emqx_ds:time(), + since := emqx_ds:time(), + until := undefined | emqx_ds:time() + } + }. +list_generations_with_lifetimes(ShardId) -> + gen_server:call(?REF(ShardId), #call_list_generations_with_lifetimes{}, infinity). + +-spec drop_generation(shard_id(), gen_id()) -> ok. +drop_generation(ShardId, GenId) -> + gen_server:call(?REF(ShardId), #call_drop_generation{gen_id = GenId}, infinity). + %%================================================================================ %% gen_server for the shard %%================================================================================ @@ -322,6 +389,13 @@ handle_call(add_generation, _From, S0) -> S = add_generation(S0, Since), commit_metadata(S), {reply, ok, S}; +handle_call(#call_list_generations_with_lifetimes{}, _From, S) -> + Generations = handle_list_generations_with_lifetimes(S), + {reply, Generations, S}; +handle_call(#call_drop_generation{gen_id = GenId}, _From, S0) -> + {Reply, S} = handle_drop_generation(S0, GenId), + commit_metadata(S), + {reply, Reply, S}; handle_call(#call_create_generation{since = Since}, _From, S0) -> S = add_generation(S0, Since), commit_metadata(S), @@ -353,7 +427,7 @@ open_shard(ShardId, DB, CFRefs, ShardSchema) -> %% Transform generation schemas to generation runtime data: maps:map( fun - ({generation, GenId}, GenSchema) -> + (?GEN_KEY(GenId), GenSchema) -> open_generation(ShardId, DB, CFRefs, GenId, GenSchema); (_K, Val) -> Val @@ -366,10 +440,40 @@ add_generation(S0, Since) -> #s{shard_id = ShardId, db = DB, schema = Schema0, shard = Shard0, cf_refs = CFRefs0} = S0, Schema1 = update_last_until(Schema0, Since), Shard1 = update_last_until(Shard0, Since), + + #{current_generation := OldGenId, prototype := {CurrentMod, _ModConf}} = Schema0, + OldKey = ?GEN_KEY(OldGenId), + #{OldKey := OldGenSchema} = Schema0, + #{cf_refs := OldCFRefs} = OldGenSchema, + #{OldKey := #{module := OldMod, data := OldGenData}} = Shard0, + {GenId, Schema, NewCFRefs} = new_generation(ShardId, DB, Schema1, Since), + CFRefs = NewCFRefs ++ CFRefs0, - Key = {generation, GenId}, - Generation = open_generation(ShardId, DB, CFRefs, GenId, maps:get(Key, Schema)), + Key = ?GEN_KEY(GenId), + Generation0 = + #{data := NewGenData0} = + open_generation(ShardId, DB, CFRefs, GenId, maps:get(Key, Schema)), + + %% When the new generation's module is the same as the last one, we might want to + %% perform actions like inheriting some of the previous (meta)data. + NewGenData = + run_post_creation_actions( + #{ + shard_id => ShardId, + db => DB, + new_gen_id => GenId, + old_gen_id => OldGenId, + new_cf_refs => NewCFRefs, + old_cf_refs => OldCFRefs, + new_gen_runtime_data => NewGenData0, + old_gen_runtime_data => OldGenData, + new_module => CurrentMod, + old_module => OldMod + } + ), + Generation = Generation0#{data := NewGenData}, + Shard = Shard1#{current_generation := GenId, Key => Generation}, S0#s{ cf_refs = CFRefs, @@ -377,6 +481,54 @@ add_generation(S0, Since) -> shard = Shard }. +-spec handle_list_generations_with_lifetimes(server_state()) -> #{gen_id() => map()}. +handle_list_generations_with_lifetimes(#s{schema = ShardSchema}) -> + maps:fold( + fun + (?GEN_KEY(GenId), GenSchema, Acc) -> + Acc#{GenId => export_generation(GenSchema)}; + (_Key, _Value, Acc) -> + Acc + end, + #{}, + ShardSchema + ). + +-spec export_generation(generation_schema()) -> map(). +export_generation(GenSchema) -> + maps:with([created_at, since, until], GenSchema). + +-spec handle_drop_generation(server_state(), gen_id()) -> + {ok | {error, current_generation}, server_state()}. +handle_drop_generation(#s{schema = #{current_generation := GenId}} = S0, GenId) -> + {{error, current_generation}, S0}; +handle_drop_generation(#s{schema = Schema} = S0, GenId) when + not is_map_key(?GEN_KEY(GenId), Schema) +-> + {{error, not_found}, S0}; +handle_drop_generation(S0, GenId) -> + #s{ + shard_id = ShardId, + db = DB, + schema = #{?GEN_KEY(GenId) := GenSchema} = OldSchema, + shard = OldShard, + cf_refs = OldCFRefs + } = S0, + #{module := Mod, cf_refs := GenCFRefs} = GenSchema, + #{?GEN_KEY(GenId) := #{data := RuntimeData}} = OldShard, + case Mod:drop(ShardId, DB, GenId, GenCFRefs, RuntimeData) of + ok -> + CFRefs = OldCFRefs -- GenCFRefs, + Shard = maps:remove(?GEN_KEY(GenId), OldShard), + Schema = maps:remove(?GEN_KEY(GenId), OldSchema), + S = S0#s{ + cf_refs = CFRefs, + shard = Shard, + schema = Schema + }, + {ok, S} + end. + -spec open_generation(shard_id(), rocksdb:db_handle(), cf_refs(), gen_id(), generation_schema()) -> generation(). open_generation(ShardId, DB, CFRefs, GenId, GenSchema) -> @@ -403,10 +555,17 @@ new_generation(ShardId, DB, Schema0, Since) -> #{current_generation := PrevGenId, prototype := {Mod, ModConf}} = Schema0, GenId = PrevGenId + 1, {GenData, NewCFRefs} = Mod:create(ShardId, DB, GenId, ModConf), - GenSchema = #{module => Mod, data => GenData, since => Since, until => undefined}, + GenSchema = #{ + module => Mod, + data => GenData, + cf_refs => NewCFRefs, + created_at => emqx_message:timestamp_now(), + since => Since, + until => undefined + }, Schema = Schema0#{ current_generation => GenId, - {generation, GenId} => GenSchema + ?GEN_KEY(GenId) => GenSchema }, {GenId, Schema, NewCFRefs}. @@ -453,9 +612,26 @@ db_dir({DB, ShardId}) -> -spec update_last_until(Schema, emqx_ds:time()) -> Schema when Schema :: shard_schema() | shard(). update_last_until(Schema, Until) -> #{current_generation := GenId} = Schema, - GenData0 = maps:get({generation, GenId}, Schema), + GenData0 = maps:get(?GEN_KEY(GenId), Schema), GenData = GenData0#{until := Until}, - Schema#{{generation, GenId} := GenData}. + Schema#{?GEN_KEY(GenId) := GenData}. + +run_post_creation_actions( + #{ + new_module := Mod, + old_module := Mod, + new_gen_runtime_data := NewGenData + } = Context +) -> + case erlang:function_exported(Mod, post_creation_actions, 1) of + true -> + Mod:post_creation_actions(Context); + false -> + NewGenData + end; +run_post_creation_actions(#{new_gen_runtime_data := NewGenData}) -> + %% Different implementation modules + NewGenData. %%-------------------------------------------------------------------------------- %% Schema access @@ -468,15 +644,24 @@ generation_current(Shard) -> -spec generation_get(shard_id(), gen_id()) -> generation(). generation_get(Shard, GenId) -> - #{{generation, GenId} := GenData} = get_schema_runtime(Shard), + {ok, GenData} = generation_get_safe(Shard, GenId), GenData. +-spec generation_get_safe(shard_id(), gen_id()) -> {ok, generation()} | {error, not_found}. +generation_get_safe(Shard, GenId) -> + case get_schema_runtime(Shard) of + #{?GEN_KEY(GenId) := GenData} -> + {ok, GenData}; + #{} -> + {error, not_found} + end. + -spec generations_since(shard_id(), emqx_ds:time()) -> [gen_id()]. generations_since(Shard, Since) -> Schema = get_schema_runtime(Shard), maps:fold( fun - ({generation, GenId}, #{until := Until}, Acc) when Until >= Since -> + (?GEN_KEY(GenId), #{until := Until}, Acc) when Until >= Since -> [GenId | Acc]; (_K, _V, Acc) -> Acc diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl index da7ac79f6..c958e56dc 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_reference.erl @@ -30,6 +30,7 @@ -export([ create/4, open/5, + drop/5, store_batch/4, get_streams/4, make_iterator/5, @@ -85,6 +86,10 @@ open(_Shard, DBHandle, GenId, CFRefs, #schema{}) -> {_, CF} = lists:keyfind(data_cf(GenId), 1, CFRefs), #s{db = DBHandle, cf = CF}. +drop(_ShardId, DBHandle, _GenId, _CFRefs, #s{cf = CFHandle}) -> + ok = rocksdb:drop_column_family(DBHandle, CFHandle), + ok. + store_batch(_ShardId, #s{db = DB, cf = CF}, Messages, _Options) -> lists:foreach( fun(Msg) -> @@ -142,7 +147,8 @@ do_next(TopicFilter, StartTime, IT, Action, NLeft, Key0, Acc) -> case rocksdb:iterator_move(IT, Action) of {ok, Key, Blob} -> Msg = #message{topic = Topic, timestamp = TS} = binary_to_term(Blob), - case emqx_topic:match(Topic, TopicFilter) andalso TS >= StartTime of + TopicWords = emqx_topic:words(Topic), + case emqx_topic:match(TopicWords, TopicFilter) andalso TS >= StartTime of true -> do_next(TopicFilter, StartTime, IT, next, NLeft - 1, Key, [{Key, Msg} | Acc]); false -> diff --git a/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v3.erl b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v3.erl new file mode 100644 index 000000000..74a174c4c --- /dev/null +++ b/apps/emqx_durable_storage/src/proto/emqx_ds_proto_v3.erl @@ -0,0 +1,147 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_ds_proto_v3). + +-behavior(emqx_bpapi). + +-include_lib("emqx_utils/include/bpapi.hrl"). +%% API: +-export([ + drop_db/2, + store_batch/5, + get_streams/5, + make_iterator/6, + next/5, + update_iterator/5, + add_generation/2, + + %% introduced in v3 + list_generations_with_lifetimes/3, + drop_generation/4 +]). + +%% behavior callbacks: +-export([introduced_in/0]). + +%%================================================================================ +%% API funcions +%%================================================================================ + +-spec drop_db([node()], emqx_ds:db()) -> + [{ok, ok} | {error, _}]. +drop_db(Node, DB) -> + erpc:multicall(Node, emqx_ds_replication_layer, do_drop_db_v1, [DB]). + +-spec get_streams( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> + [{integer(), emqx_ds_storage_layer:stream()}]. +get_streams(Node, DB, Shard, TopicFilter, Time) -> + erpc:call(Node, emqx_ds_replication_layer, do_get_streams_v1, [DB, Shard, TopicFilter, Time]). + +-spec make_iterator( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:stream(), + emqx_ds:topic_filter(), + emqx_ds:time() +) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _}. +make_iterator(Node, DB, Shard, Stream, TopicFilter, StartTime) -> + erpc:call(Node, emqx_ds_replication_layer, do_make_iterator_v1, [ + DB, Shard, Stream, TopicFilter, StartTime + ]). + +-spec next( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:iterator(), + pos_integer() +) -> + {ok, emqx_ds_storage_layer:iterator(), [{emqx_ds:message_key(), [emqx_types:message()]}]} + | {ok, end_of_stream} + | {error, _}. +next(Node, DB, Shard, Iter, BatchSize) -> + emqx_rpc:call(Shard, Node, emqx_ds_replication_layer, do_next_v1, [DB, Shard, Iter, BatchSize]). + +-spec store_batch( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_replication_layer:batch(), + emqx_ds:message_store_opts() +) -> + emqx_ds:store_batch_result(). +store_batch(Node, DB, Shard, Batch, Options) -> + emqx_rpc:call(Shard, Node, emqx_ds_replication_layer, do_store_batch_v1, [ + DB, Shard, Batch, Options + ]). + +-spec update_iterator( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:iterator(), + emqx_ds:message_key() +) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _}. +update_iterator(Node, DB, Shard, OldIter, DSKey) -> + erpc:call(Node, emqx_ds_replication_layer, do_update_iterator_v2, [ + DB, Shard, OldIter, DSKey + ]). + +-spec add_generation([node()], emqx_ds:db()) -> + [{ok, ok} | {error, _}]. +add_generation(Node, DB) -> + erpc:multicall(Node, emqx_ds_replication_layer, do_add_generation_v2, [DB]). + +%%-------------------------------------------------------------------------------- +%% Introduced in V3 +%%-------------------------------------------------------------------------------- + +-spec list_generations_with_lifetimes( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id() +) -> + #{ + emqx_ds:ds_specific_generation_rank() => emqx_ds:generation_info() + }. +list_generations_with_lifetimes(Node, DB, Shard) -> + erpc:call(Node, emqx_ds_replication_layer, do_list_generations_with_lifetimes_v3, [DB, Shard]). + +-spec drop_generation( + node(), + emqx_ds:db(), + emqx_ds_replication_layer:shard_id(), + emqx_ds_storage_layer:gen_id() +) -> + ok | {error, _}. +drop_generation(Node, DB, Shard, GenId) -> + erpc:call(Node, emqx_ds_replication_layer, do_drop_generation_v3, [DB, Shard, GenId]). + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +introduced_in() -> + "5.6.0". diff --git a/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl index cb9d81580..d7dccccf5 100644 --- a/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl +++ b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl @@ -155,7 +155,7 @@ t_05_update_iterator(_Config) -> ?assertEqual(Msgs, AllMsgs, #{from_key => Iter1, final_iter => FinalIter}), ok. -t_05_update_config(_Config) -> +t_06_update_config(_Config) -> DB = ?FUNCTION_NAME, ?assertMatch(ok, emqx_ds:open_db(DB, opts())), TopicFilter = ['#'], @@ -199,7 +199,7 @@ t_05_update_config(_Config) -> end, lists:foldl(Checker, [], lists:zip(StartTimes, MsgsList)). -t_06_add_generation(_Config) -> +t_07_add_generation(_Config) -> DB = ?FUNCTION_NAME, ?assertMatch(ok, emqx_ds:open_db(DB, opts())), TopicFilter = ['#'], @@ -243,6 +243,250 @@ t_06_add_generation(_Config) -> end, lists:foldl(Checker, [], lists:zip(StartTimes, MsgsList)). +%% Verifies the basic usage of `list_generations_with_lifetimes' and `drop_generation'... +%% 1) Cannot drop current generation. +%% 2) All existing generations are returned by `list_generation_with_lifetimes'. +%% 3) Dropping a generation removes it from the list. +%% 4) Dropped generations stay dropped even after restarting the application. +t_08_smoke_list_drop_generation(_Config) -> + DB = ?FUNCTION_NAME, + ?check_trace( + begin + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + %% Exactly one generation at first. + Generations0 = emqx_ds:list_generations_with_lifetimes(DB), + ?assertMatch( + [{_GenId, #{since := _, until := _}}], + maps:to_list(Generations0), + #{gens => Generations0} + ), + [{GenId0, _}] = maps:to_list(Generations0), + %% Cannot delete current generation + ?assertEqual({error, current_generation}, emqx_ds:drop_generation(DB, GenId0)), + + %% New gen + ok = emqx_ds:add_generation(DB), + Generations1 = emqx_ds:list_generations_with_lifetimes(DB), + ?assertMatch( + [ + {GenId0, #{since := _, until := _}}, + {_GenId1, #{since := _, until := _}} + ], + lists:sort(maps:to_list(Generations1)), + #{gens => Generations1} + ), + [GenId0, GenId1] = lists:sort(maps:keys(Generations1)), + + %% Drop the older one + ?assertEqual(ok, emqx_ds:drop_generation(DB, GenId0)), + Generations2 = emqx_ds:list_generations_with_lifetimes(DB), + ?assertMatch( + [{GenId1, #{since := _, until := _}}], + lists:sort(maps:to_list(Generations2)), + #{gens => Generations2} + ), + + %% Unknown gen_id, as it was already dropped + ?assertEqual({error, not_found}, emqx_ds:drop_generation(DB, GenId0)), + + %% Should persist surviving generation list + ok = application:stop(emqx_durable_storage), + {ok, _} = application:ensure_all_started(emqx_durable_storage), + ok = emqx_ds:open_db(DB, opts()), + + Generations3 = emqx_ds:list_generations_with_lifetimes(DB), + ?assertMatch( + [{GenId1, #{since := _, until := _}}], + lists:sort(maps:to_list(Generations3)), + #{gens => Generations3} + ), + + ok + end, + [] + ), + ok. + +t_drop_generation_with_never_used_iterator(_Config) -> + %% This test checks how the iterator behaves when: + %% 1) it's created at generation 1 and not consumed from. + %% 2) generation 2 is created and 1 dropped. + %% 3) iteration begins. + %% In this case, the iterator won't see any messages and the stream will end. + + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + [GenId0] = maps:keys(emqx_ds:list_generations_with_lifetimes(DB)), + + TopicFilter = emqx_topic:words(<<"foo/+">>), + StartTime = 0, + Msgs0 = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo/baz">>, <<"2">>, 1) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs0)), + + [{_, Stream0}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + {ok, Iter0} = emqx_ds:make_iterator(DB, Stream0, TopicFilter, StartTime), + + ok = emqx_ds:add_generation(DB), + ok = emqx_ds:drop_generation(DB, GenId0), + + Now = emqx_message:timestamp_now(), + Msgs1 = [ + message(<<"foo/bar">>, <<"3">>, Now + 100), + message(<<"foo/baz">>, <<"4">>, Now + 101) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs1)), + + ?assertMatch({ok, end_of_stream, []}, iterate(DB, Iter0, 1)), + + %% New iterator for the new stream will only see the later messages. + [{_, Stream1}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + ?assertNotEqual(Stream0, Stream1), + {ok, Iter1} = emqx_ds:make_iterator(DB, Stream1, TopicFilter, StartTime), + + {ok, Iter, Batch} = iterate(DB, Iter1, 1), + ?assertNotEqual(end_of_stream, Iter), + ?assertEqual(Msgs1, [Msg || {_Key, Msg} <- Batch]), + + ok. + +t_drop_generation_with_used_once_iterator(_Config) -> + %% This test checks how the iterator behaves when: + %% 1) it's created at generation 1 and consumes at least 1 message. + %% 2) generation 2 is created and 1 dropped. + %% 3) iteration continues. + %% In this case, the iterator should see no more messages and the stream will end. + + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + [GenId0] = maps:keys(emqx_ds:list_generations_with_lifetimes(DB)), + + TopicFilter = emqx_topic:words(<<"foo/+">>), + StartTime = 0, + Msgs0 = + [Msg0 | _] = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo/baz">>, <<"2">>, 1) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs0)), + + [{_, Stream0}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + {ok, Iter0} = emqx_ds:make_iterator(DB, Stream0, TopicFilter, StartTime), + {ok, Iter1, Batch1} = emqx_ds:next(DB, Iter0, 1), + ?assertNotEqual(end_of_stream, Iter1), + ?assertEqual([Msg0], [Msg || {_Key, Msg} <- Batch1]), + + ok = emqx_ds:add_generation(DB), + ok = emqx_ds:drop_generation(DB, GenId0), + + Now = emqx_message:timestamp_now(), + Msgs1 = [ + message(<<"foo/bar">>, <<"3">>, Now + 100), + message(<<"foo/baz">>, <<"4">>, Now + 101) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs1)), + + ?assertMatch({ok, end_of_stream, []}, iterate(DB, Iter1, 1)), + + ok. + +t_drop_generation_update_iterator(_Config) -> + %% This checks the behavior of `emqx_ds:update_iterator' after the generation + %% underlying the iterator has been dropped. + + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + [GenId0] = maps:keys(emqx_ds:list_generations_with_lifetimes(DB)), + + TopicFilter = emqx_topic:words(<<"foo/+">>), + StartTime = 0, + Msgs0 = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo/baz">>, <<"2">>, 1) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs0)), + + [{_, Stream0}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + {ok, Iter0} = emqx_ds:make_iterator(DB, Stream0, TopicFilter, StartTime), + {ok, Iter1, _Batch1} = emqx_ds:next(DB, Iter0, 1), + {ok, _Iter2, [{Key2, _Msg}]} = emqx_ds:next(DB, Iter1, 1), + + ok = emqx_ds:add_generation(DB), + ok = emqx_ds:drop_generation(DB, GenId0), + + ?assertEqual({error, end_of_stream}, emqx_ds:update_iterator(DB, Iter1, Key2)), + + ok. + +t_make_iterator_stale_stream(_Config) -> + %% This checks the behavior of `emqx_ds:make_iterator' after the generation underlying + %% the stream has been dropped. + + DB = ?FUNCTION_NAME, + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + [GenId0] = maps:keys(emqx_ds:list_generations_with_lifetimes(DB)), + + TopicFilter = emqx_topic:words(<<"foo/+">>), + StartTime = 0, + Msgs0 = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo/baz">>, <<"2">>, 1) + ], + ?assertMatch(ok, emqx_ds:store_batch(DB, Msgs0)), + + [{_, Stream0}] = emqx_ds:get_streams(DB, TopicFilter, StartTime), + + ok = emqx_ds:add_generation(DB), + ok = emqx_ds:drop_generation(DB, GenId0), + + ?assertEqual( + {error, end_of_stream}, + emqx_ds:make_iterator(DB, Stream0, TopicFilter, StartTime) + ), + + ok. + +t_get_streams_concurrently_with_drop_generation(_Config) -> + %% This checks that we can get all streams while a generation is dropped + %% mid-iteration. + + DB = ?FUNCTION_NAME, + ?check_trace( + #{timetrap => 5_000}, + begin + ?assertMatch(ok, emqx_ds:open_db(DB, opts())), + + [GenId0] = maps:keys(emqx_ds:list_generations_with_lifetimes(DB)), + ok = emqx_ds:add_generation(DB), + ok = emqx_ds:add_generation(DB), + + %% All streams + TopicFilter = emqx_topic:words(<<"foo/+">>), + StartTime = 0, + ?assertMatch([_, _, _], emqx_ds:get_streams(DB, TopicFilter, StartTime)), + + ?force_ordering( + #{?snk_kind := dropped_gen}, + #{?snk_kind := get_streams_get_gen} + ), + + spawn_link(fun() -> + {ok, _} = ?block_until(#{?snk_kind := get_streams_all_gens}), + ok = emqx_ds:drop_generation(DB, GenId0), + ?tp(dropped_gen, #{}) + end), + + ?assertMatch([_, _], emqx_ds:get_streams(DB, TopicFilter, StartTime)), + + ok + end, + [] + ), + + ok. + update_data_set() -> [ [ @@ -295,7 +539,7 @@ iterate(DB, It0, BatchSize, Acc) -> {ok, It, Msgs} -> iterate(DB, It, BatchSize, Acc ++ Msgs); {ok, end_of_stream} -> - {ok, It0, Acc}; + {ok, end_of_stream, Acc}; Ret -> Ret end. From db710c4be59914edc987bf951e85d62960d6593e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 Jan 2024 14:28:13 -0300 Subject: [PATCH 16/32] feat(lts): inherit previous generation's lts when possible --- apps/emqx_durable_storage/src/emqx_ds_lts.erl | 11 ++++++- .../src/emqx_ds_storage_bitfield_lts.erl | 30 ++++++++++++++++- .../emqx_ds_storage_bitfield_lts_SUITE.erl | 33 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_lts.erl index bcf95852d..9d87cf571 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_lts.erl @@ -18,7 +18,12 @@ %% API: -export([ - trie_create/1, trie_create/0, trie_restore/2, topic_key/3, match_topics/2, lookup_topic_key/2 + trie_create/1, trie_create/0, + trie_restore/2, + trie_restore_existing/2, + topic_key/3, + match_topics/2, + lookup_topic_key/2 ]). %% Debug: @@ -115,6 +120,10 @@ trie_create() -> -spec trie_restore(options(), [{_Key, _Val}]) -> trie(). trie_restore(Options, Dump) -> Trie = trie_create(Options), + trie_restore_existing(Trie, Dump). + +-spec trie_restore_existing(trie(), [{_Key, _Val}]) -> trie(). +trie_restore_existing(Trie, Dump) -> lists:foreach( fun({{StateFrom, Token}, StateTo}) -> trie_insert(Trie, StateFrom, Token, StateTo) diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl index 27d41e6c6..2a3086a57 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl @@ -32,7 +32,8 @@ get_streams/4, make_iterator/5, update_iterator/4, - next/4 + next/4, + post_creation_actions/1 ]). %% internal exports: @@ -200,6 +201,22 @@ open(_Shard, DBHandle, GenId, CFRefs, Schema) -> ts_offset = TSOffsetBits }. +-spec post_creation_actions(emqx_ds_storage_layer:post_creation_context()) -> + s(). +post_creation_actions( + #{ + db := DBHandle, + old_gen_id := OldGenId, + old_cf_refs := OldCFRefs, + new_gen_runtime_data := NewGenData0 + } +) -> + {_, OldTrieCF} = lists:keyfind(trie_cf(OldGenId), 1, OldCFRefs), + #s{trie = NewTrie0} = NewGenData0, + NewTrie = copy_previous_trie(DBHandle, NewTrie0, OldTrieCF), + ?tp(bitfield_lts_inherited_trie, #{}), + NewGenData0#s{trie = NewTrie}. + -spec drop( emqx_ds_storage_layer:shard_id(), rocksdb:db_handle(), @@ -516,6 +533,17 @@ restore_trie(TopicIndexBytes, DB, CF) -> rocksdb:iterator_close(IT) end. +-spec copy_previous_trie(rocksdb:db_handle(), emqx_ds_lts:trie(), rocksdb:cf_handle()) -> + emqx_ds_lts:trie(). +copy_previous_trie(DBHandle, NewTrie, OldCF) -> + {ok, IT} = rocksdb:iterator(DBHandle, OldCF, []), + try + OldDump = read_persisted_trie(IT, rocksdb:iterator_move(IT, first)), + emqx_ds_lts:trie_restore_existing(NewTrie, OldDump) + after + rocksdb:iterator_close(IT) + end. + read_persisted_trie(IT, {ok, KeyB, ValB}) -> [ {binary_to_term(KeyB), binary_to_term(ValB)} diff --git a/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl index 03ff1a6cb..5d32143a7 100644 --- a/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl +++ b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl @@ -131,6 +131,39 @@ t_get_streams(_Config) -> ?assert(lists:member(A, AllStreams)), ok. +t_new_generation_inherit_trie(_Config) -> + %% This test checks that we inherit the previous generation's LTS when creating a new + %% generation. + ?check_trace( + begin + %% Create a bunch of topics to be learned in the first generation + Timestamps = lists:seq(1, 10_000, 100), + Batch = [ + begin + B = integer_to_binary(I), + make_message( + TS, + <<"wildcard/", B/binary, "/suffix/", Suffix/binary>>, + integer_to_binary(TS) + ) + end + || I <- lists:seq(1, 200), + TS <- Timestamps, + Suffix <- [<<"foo">>, <<"bar">>] + ], + ok = emqx_ds_storage_layer:store_batch(?SHARD, Batch, []), + %% Now we create a new generation with the same LTS module. It should inherit the + %% learned trie. + ok = emqx_ds_storage_layer:add_generation(?SHARD), + ok + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(bitfield_lts_inherited_trie, Trace)), + ok + end + ), + ok. + t_replay(_Config) -> %% Create concrete topics: Topics = [<<"foo/bar">>, <<"foo/bar/baz">>], From 7035b4c8b3fc6c8ced3d0b623bc7144f80348044 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 18 Jan 2024 14:28:37 -0300 Subject: [PATCH 17/32] feat(ps): add message gc --- .../emqx_persistent_message_ds_gc_worker.erl | 157 ++++++++++++++++++ .../src/emqx_persistent_session_ds_sup.erl | 5 +- apps/emqx/src/emqx_schema.erl | 8 + .../test/emqx_persistent_messages_SUITE.erl | 87 +++++++++- changes/ce/feat-12338.en.md | 1 + rel/i18n/emqx_schema.hocon | 3 + 6 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl create mode 100644 changes/ce/feat-12338.en.md diff --git a/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl new file mode 100644 index 000000000..b960eae9e --- /dev/null +++ b/apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl @@ -0,0 +1,157 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_persistent_message_ds_gc_worker). + +-behaviour(gen_server). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +-include("emqx_persistent_session_ds.hrl"). + +%% API +-export([ + start_link/0, + gc/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%% call/cast/info records +-record(gc, {}). + +%%-------------------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% For testing or manual ops +gc() -> + gen_server:call(?MODULE, #gc{}, infinity). + +%%-------------------------------------------------------------------------------- +%% `gen_server' API +%%-------------------------------------------------------------------------------- + +init(_Opts) -> + ensure_gc_timer(), + State = #{}, + {ok, State}. + +handle_call(#gc{}, _From, State) -> + maybe_gc(), + {reply, ok, State}; +handle_call(_Call, _From, State) -> + {reply, error, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info(#gc{}, State) -> + try_gc(), + ensure_gc_timer(), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------------------- +%% Internal fns +%%-------------------------------------------------------------------------------- + +ensure_gc_timer() -> + Timeout = emqx_config:get([session_persistence, message_retention_period]), + _ = erlang:send_after(Timeout, self(), #gc{}), + ok. + +try_gc() -> + %% Only cores should run GC. + CoreNodes = mria_membership:running_core_nodelist(), + Res = global:trans( + {?MODULE, self()}, + fun maybe_gc/0, + CoreNodes, + %% Note: we set retries to 1 here because, in rare occasions, GC might start at the + %% same time in more than one node, and each one will abort the other. By allowing + %% one retry, at least one node will (hopefully) get to enter the transaction and + %% the other will abort. If GC runs too fast, both nodes might run in sequence. + %% But, in that case, GC is clearly not too costly, and that shouldn't be a problem, + %% resource-wise. + _Retries = 1 + ), + case Res of + aborted -> + ?tp(ds_message_gc_lock_taken, #{}), + ok; + ok -> + ok + end. + +now_ms() -> + erlang:system_time(millisecond). + +maybe_gc() -> + AllGens = emqx_ds:list_generations_with_lifetimes(?PERSISTENT_MESSAGE_DB), + NowMS = now_ms(), + RetentionPeriod = emqx_config:get([session_persistence, message_retention_period]), + TimeThreshold = NowMS - RetentionPeriod, + maybe_create_new_generation(AllGens, TimeThreshold), + ?tp_span( + ps_message_gc, + #{}, + begin + ExpiredGens = + maps:filter( + fun(_GenId, #{until := Until}) -> + is_number(Until) andalso Until =< TimeThreshold + end, + AllGens + ), + ExpiredGenIds = maps:keys(ExpiredGens), + lists:foreach( + fun(GenId) -> + ok = emqx_ds:drop_generation(?PERSISTENT_MESSAGE_DB, GenId), + ?tp(message_gc_generation_dropped, #{gen_id => GenId}) + end, + ExpiredGenIds + ) + end + ). + +maybe_create_new_generation(AllGens, TimeThreshold) -> + NeedNewGen = + lists:all( + fun({_GenId, #{created_at := CreatedAt}}) -> + CreatedAt =< TimeThreshold + end, + maps:to_list(AllGens) + ), + case NeedNewGen of + false -> + ?tp(ps_message_gc_too_early, #{}), + ok; + true -> + ok = emqx_ds:add_generation(?PERSISTENT_MESSAGE_DB), + ?tp(ps_message_gc_added_gen, #{}) + end. diff --git a/apps/emqx/src/emqx_persistent_session_ds_sup.erl b/apps/emqx/src/emqx_persistent_session_ds_sup.erl index 5bd620e8b..11e05be82 100644 --- a/apps/emqx/src/emqx_persistent_session_ds_sup.erl +++ b/apps/emqx/src/emqx_persistent_session_ds_sup.erl @@ -48,13 +48,14 @@ init(Opts) -> do_init(_Opts) -> SupFlags = #{ - strategy => rest_for_one, + strategy => one_for_one, intensity => 10, period => 2, auto_shutdown => never }, CoreChildren = [ - worker(gc_worker, emqx_persistent_session_ds_gc_worker, []) + worker(session_gc_worker, emqx_persistent_session_ds_gc_worker, []), + worker(message_gc_worker, emqx_persistent_message_ds_gc_worker, []) ], Children = case mria_rlog:role() of diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 1dd0a55ed..610c07d49 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1853,6 +1853,14 @@ fields("session_persistence") -> desc => ?DESC(session_ds_session_gc_batch_size) } )}, + {"message_retention_period", + sc( + timeout_duration(), + #{ + default => <<"1d">>, + desc => ?DESC(session_ds_message_retention_period) + } + )}, {"force_persistence", sc( boolean(), diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index f25f38098..c46d726f4 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -19,6 +19,7 @@ -include_lib("stdlib/include/assert.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -compile(export_all). @@ -45,10 +46,20 @@ init_per_testcase(t_session_subscription_iterators = TestCase, Config) -> Cluster = cluster(), Nodes = emqx_cth_cluster:start(Cluster, #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}), [{nodes, Nodes} | Config]; +init_per_testcase(t_message_gc = TestCase, Config) -> + Opts = #{ + extra_emqx_conf => + "\n session_persistence.message_retention_period = 1s" + "\n session_persistence.storage.builtin.n_shards = 3" + }, + common_init_per_testcase(TestCase, [{n_shards, 3} | Config], Opts); init_per_testcase(TestCase, Config) -> + common_init_per_testcase(TestCase, Config, _Opts = #{}). + +common_init_per_testcase(TestCase, Config, Opts) -> ok = emqx_ds:drop_db(?PERSISTENT_MESSAGE_DB), Apps = emqx_cth_suite:start( - app_specs(), + app_specs(Opts), #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} ), [{apps, Apps} | Config]. @@ -379,6 +390,66 @@ t_publish_empty_topic_levels(_Config) -> emqtt:stop(Pub) end. +t_message_gc_too_young(_Config) -> + %% Check that GC doesn't attempt to create a new generation if there are fresh enough + %% generations around. The stability of this test relies on the default value for + %% message retention being long enough. Currently, the default is 1 hour. + ?check_trace( + ok = emqx_persistent_message_ds_gc_worker:gc(), + fun(Trace) -> + ?assertMatch([_], ?of_kind(ps_message_gc_too_early, Trace)), + ok + end + ), + ok. + +t_message_gc(Config) -> + %% Check that, after GC runs, a new generation is created, retaining messages, and + %% older messages no longer are accessible. + NShards = ?config(n_shards, Config), + ?check_trace( + #{timetrap => 10_000}, + begin + %% ensure some messages are in the first generation + ?force_ordering( + #{?snk_kind := inserted_batch}, + #{?snk_kind := ps_message_gc_added_gen} + ), + Msgs0 = [ + message(<<"foo/bar">>, <<"1">>, 0), + message(<<"foo/baz">>, <<"2">>, 1) + ], + ok = emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, Msgs0), + ?tp(inserted_batch, #{}), + {ok, _} = ?block_until(#{?snk_kind := ps_message_gc_added_gen}), + + Now = emqx_message:timestamp_now(), + Msgs1 = [ + message(<<"foo/bar">>, <<"3">>, Now + 100), + message(<<"foo/baz">>, <<"4">>, Now + 101) + ], + ok = emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, Msgs1), + + {ok, _} = snabbkaffe:block_until( + ?match_n_events(NShards, #{?snk_kind := message_gc_generation_dropped}), + infinity + ), + + TopicFilter = emqx_topic:words(<<"#">>), + StartTime = 0, + Msgs = consume(TopicFilter, StartTime), + %% only "1" and "2" should have been GC'ed + ?assertEqual( + sets:from_list([<<"3">>, <<"4">>], [{version, 2}]), + sets:from_list([emqx_message:payload(Msg) || Msg <- Msgs], [{version, 2}]) + ), + + ok + end, + [] + ), + ok. + %% connect(ClientId, CleanStart, EI) -> @@ -438,9 +509,13 @@ publish(Node, Message) -> erpc:call(Node, emqx, publish, [Message]). app_specs() -> + app_specs(_Opts = #{}). + +app_specs(Opts) -> + ExtraEMQXConf = maps:get(extra_emqx_conf, Opts, ""), [ emqx_durable_storage, - {emqx, "session_persistence {enable = true}"} + {emqx, "session_persistence {enable = true}" ++ ExtraEMQXConf} ]. cluster() -> @@ -459,3 +534,11 @@ clear_db() -> mria:stop(), ok = mnesia:delete_schema([node()]), ok. + +message(Topic, Payload, PublishedAt) -> + #message{ + topic = Topic, + payload = Payload, + timestamp = PublishedAt, + id = emqx_guid:gen() + }. diff --git a/changes/ce/feat-12338.en.md b/changes/ce/feat-12338.en.md new file mode 100644 index 000000000..8b8edcb76 --- /dev/null +++ b/changes/ce/feat-12338.en.md @@ -0,0 +1 @@ +Added time-based message garbage collection to the RocksDB-based persistent session backend. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index fe315b5d7..75dce469e 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1603,5 +1603,8 @@ The session will query the DB for the new messages when the value of `FreeSpace` `FreeSpace` is calculated as `ReceiveMaximum` for the session - number of inflight messages.""" +session_ds_message_retention_period.desc: +"""The minimum amount of time that messages should be retained for. After messages have been in storage for at least this period of time, they'll be dropped.""" + } From 609ba7e3327b60267a24e76334247151aff2fdc6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Jan 2024 13:10:44 -0300 Subject: [PATCH 18/32] fix(ds): do not count persistent session-only routed messages as dropped Fixes https://emqx.atlassian.net/browse/EMQX-11539 --- apps/emqx/src/emqx_broker.erl | 12 ++++--- .../test/emqx_persistent_messages_SUITE.erl | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index ac9116cbd..6dc893043 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -249,7 +249,7 @@ publish(Msg) when is_record(Msg, message) -> []; Msg1 = #message{topic = Topic} -> PersistRes = persist_publish(Msg1), - PersistRes ++ route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)) + route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1), PersistRes) end. persist_publish(Msg) -> @@ -289,18 +289,20 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}. %% Route %%-------------------------------------------------------------------- --spec route([emqx_types:route_entry()], emqx_types:delivery()) -> +-spec route([emqx_types:route_entry()], emqx_types:delivery(), nil() | [persisted]) -> emqx_types:publish_result(). -route([], #delivery{message = Msg}) -> +route([], #delivery{message = Msg}, _PersistRes = []) -> ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]), ok = inc_dropped_cnt(Msg), []; -route(Routes, Delivery) -> +route([], _Delivery, PersistRes = [_ | _]) -> + PersistRes; +route(Routes, Delivery, PersistRes) -> lists:foldl( fun(Route, Acc) -> [do_route(Route, Delivery) | Acc] end, - [], + PersistRes, Routes ). diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index c46d726f4..73c88adc8 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -450,6 +450,32 @@ t_message_gc(Config) -> ), ok. +t_metrics_not_dropped(_Config) -> + %% Asserts that, if only persisted sessions are subscribed to a topic being published + %% to, we don't bump the `message.dropped' metric, nor we run the equivalent hook. + Sub = connect(<>, true, 30), + on_exit(fun() -> emqtt:stop(Sub) end), + Pub = connect(<>, true, 30), + on_exit(fun() -> emqtt:stop(Pub) end), + Hookpoint = 'message.dropped', + emqx_hooks:add(Hookpoint, {?MODULE, on_message_dropped, [self()]}, 1_000), + on_exit(fun() -> emqx_hooks:del(Hookpoint, {?MODULE, on_message_dropped}) end), + + DroppedBefore = emqx_metrics:val('messages.dropped'), + DroppedNoSubBefore = emqx_metrics:val('messages.dropped.no_subscribers'), + + {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Sub, <<"t/+">>, ?QOS_1), + emqtt:publish(Pub, <<"t/ps">>, <<"payload">>, ?QOS_1), + ?assertMatch([_], receive_messages(1, 1_500)), + + DroppedAfter = emqx_metrics:val('messages.dropped'), + DroppedNoSubAfter = emqx_metrics:val('messages.dropped.no_subscribers'), + + ?assertEqual(DroppedBefore, DroppedAfter), + ?assertEqual(DroppedNoSubBefore, DroppedNoSubAfter), + + ok. + %% connect(ClientId, CleanStart, EI) -> @@ -542,3 +568,8 @@ message(Topic, Payload, PublishedAt) -> timestamp = PublishedAt, id = emqx_guid:gen() }. + +on_message_dropped(Msg, Context, Res, TestPid) -> + ErrCtx = #{msg => Msg, ctx => Context, res => Res}, + ct:pal("this hook should not be called.\n ~p", [ErrCtx]), + exit(TestPid, {hookpoint_called, ErrCtx}). From 28867d07e69fb1db66b9ab095390db2f42a0ffd8 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 23 Jan 2024 10:00:56 +0800 Subject: [PATCH 19/32] fix(opentsdb): Enhanced the type support for template data --- .../src/emqx_bridge_opents.erl | 18 +++- .../src/emqx_bridge_opents_connector.erl | 27 ++++-- .../test/emqx_bridge_opents_SUITE.erl | 88 ++++++++++++++++++- rel/i18n/emqx_bridge_opents.hocon | 2 +- 4 files changed, 124 insertions(+), 11 deletions(-) diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl index 7e490576f..119de1978 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl @@ -3,6 +3,7 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_opents). +-include_lib("emqx/include/logger.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -156,12 +157,25 @@ fields(action_parameters_data) -> binary(), #{ required => true, - desc => ?DESC("config_parameters_tags") + desc => ?DESC("config_parameters_tags"), + validator => fun(Tmpl) -> + case emqx_placeholder:preproc_tmpl(Tmpl) of + [{var, _}] -> + true; + _ -> + ?SLOG(warning, #{ + msg => "invalid_tags_template", + path => "opents.parameters.data.tags", + data => Tmpl + }), + false + end + end } )}, {value, mk( - binary(), + hoconsc:union([integer(), float(), binary()]), #{ required => true, desc => ?DESC("config_parameters_value") diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index e3fe9d6b4..d71468d82 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -304,9 +304,14 @@ render_channel_message(Msg, #{data := DataList}, Acc) -> ValueVal = case ValueTk of [_] -> + %% just one element, maybe is a variable or a plain text + %% we should keep it as it is erlang:hd(emqx_placeholder:proc_tmpl(ValueTk, Msg, RawOpts)); - _ -> - emqx_placeholder:proc_tmpl(ValueTk, Msg) + Tks when is_list(Tks) -> + emqx_placeholder:proc_tmpl(ValueTk, Msg); + Raw -> + %% not a token list, just a raw value + Raw end, Base = #{metric => MetricVal, tags => TagsVal, value => ValueVal}, [ @@ -328,12 +333,20 @@ preproc_data_template([]) -> preproc_data_template(DataList) -> lists:map( fun(Data) -> - maps:map( - fun(_Key, Value) -> - emqx_placeholder:preproc_tmpl(Value) + {Value, Data2} = maps:take(value, Data), + Template = maps:map( + fun(_Key, Val) -> + emqx_placeholder:preproc_tmpl(Val) end, - Data - ) + Data2 + ), + + case Value of + Text when is_binary(Text) -> + Template#{value => emqx_placeholder:preproc_tmpl(Text)}; + Raw -> + Template#{value => Raw} + end end, DataList ). diff --git a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl index f86ae6986..e3e89d563 100644 --- a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl +++ b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl @@ -83,7 +83,7 @@ init_per_testcase(TestCase, Config0) -> {bridge_config, ActionConfig} | Config0 ], - %% iotdb_reset(Config), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), ok = snabbkaffe:start_trace(), Config. @@ -253,3 +253,89 @@ t_query_invalid_data(Config) -> ok = emqx_bridge_v2_testlib:t_sync_query( Config, MakeMessageFun, fun is_error_check/1, opents_bridge_on_query ). + +t_tags_validator(Config) -> + %% Create without data configured + ?assertMatch({ok, _}, emqx_bridge_v2_testlib:create_bridge(Config)), + + ?assertMatch( + {ok, _}, + emqx_bridge_v2_testlib:update_bridge_api(Config, #{ + <<"parameters">> => #{ + <<"data">> => [ + #{ + <<"metric">> => <<"${metric}">>, + <<"tags">> => <<"${tags}">>, + <<"value">> => <<"${payload.value}">> + } + ] + } + }) + ), + + ?assertMatch( + {error, _}, + emqx_bridge_v2_testlib:update_bridge_api(Config, #{ + <<"parameters">> => #{ + <<"data">> => [ + #{ + <<"metric">> => <<"${metric}">>, + <<"tags">> => <<"text">>, + <<"value">> => <<"${payload.value}">> + } + ] + } + }) + ). + +t_raw_int_value(Config) -> + raw_value_test(<<"t_raw_int_value">>, 42, Config). + +t_raw_float_value(Config) -> + raw_value_test(<<"t_raw_float_value">>, 42.5, Config). + +raw_value_test(Metric, RawValue, Config) -> + ?assertMatch({ok, _}, emqx_bridge_v2_testlib:create_bridge(Config)), + ResourceId = emqx_bridge_v2_testlib:resource_id(Config), + BridgeId = emqx_bridge_v2_testlib:bridge_id(Config), + ?retry( + _Sleep = 1_000, + _Attempts = 10, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + ?assertMatch( + {ok, _}, + emqx_bridge_v2_testlib:update_bridge_api(Config, #{ + <<"parameters">> => #{ + <<"data">> => [ + #{ + <<"metric">> => <<"${metric}">>, + <<"tags">> => <<"${tags}">>, + <<"value">> => RawValue + } + ] + } + }) + ), + + Value = 12, + MakeMessageFun = fun() -> make_data(Metric, Value) end, + + is_success_check( + emqx_resource:simple_sync_query(ResourceId, {BridgeId, MakeMessageFun()}) + ), + + {ok, {{_, 200, _}, _, IoTDBResult}} = opentds_query(Config, Metric), + QResult = emqx_utils_json:decode(IoTDBResult), + ?assertMatch( + [ + #{ + <<"metric">> := Metric, + <<"dps">> := _ + } + ], + QResult + ), + [#{<<"dps">> := Dps}] = QResult, + ?assertMatch([RawValue | _], maps:values(Dps)). diff --git a/rel/i18n/emqx_bridge_opents.hocon b/rel/i18n/emqx_bridge_opents.hocon index f5d2ade85..ab2e82180 100644 --- a/rel/i18n/emqx_bridge_opents.hocon +++ b/rel/i18n/emqx_bridge_opents.hocon @@ -49,7 +49,7 @@ config_parameters_metric.label: """Metric""" config_parameters_tags.desc: -"""Data Type, Placeholders in format of ${var} is supported""" +"""Tags. Only supports with placeholder to extract tags from a variable""" config_parameters_tags.label: """Tags""" From 0dbaaa5d949ab4235cf360ae6594b54b335f69d1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 23 Jan 2024 09:06:15 +0800 Subject: [PATCH 20/32] chore: document api-key path api only support bearerAuth --- apps/emqx/include/http_api.hrl | 1 + apps/emqx_dashboard/src/emqx_dashboard.erl | 7 ++++--- apps/emqx_dashboard/src/emqx_dashboard_api.erl | 1 + .../src/emqx_mgmt_api_api_keys.erl | 2 ++ apps/emqx_management/src/emqx_mgmt_auth.erl | 6 +++--- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 17 ++++++++++++++++- rel/i18n/emqx_mgmt_api_api_keys.hocon | 10 +++++----- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 0f6372584..f0c5611e9 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -17,6 +17,7 @@ %% HTTP API Auth -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET'). +-define(API_KEY_NOT_ALLOW, 'API_KEY_NOT_ALLOW'). -define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>). %% Bad Request diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 96f81ca84..a4438f6c7 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -264,10 +264,11 @@ api_key_authorize(Req, Key, Secret) -> case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of ok -> {ok, #{auth_type => api_key, source => Key}}; - {error, <<"not_allowed">>} -> + {error, <<"not_allowed">>, Resource} -> return_unauthorized( - ?BAD_API_KEY_OR_SECRET, - <<"Not allowed, Check api_key/api_secret">> + ?API_KEY_NOT_ALLOW, + <<"Please use bearer Token instead, using API key/secret in ", Resource/binary, + " path is not permitted">> ); {error, unauthorized_role} -> {403, 'UNAUTHORIZED_ROLE', ?API_KEY_NOT_ALLOW_MSG}; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 8a81f2116..d7ed5941f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -89,6 +89,7 @@ schema("/logout") -> post => #{ tags => [<<"dashboard">>], desc => ?DESC(logout_api), + security => [#{'bearerAuth' => []}], parameters => sso_parameters(), 'requestBody' => fields([username]), responses => #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index db20e9477..ad4b53401 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -40,6 +40,7 @@ schema("/api_key") -> get => #{ description => ?DESC(api_key_list), tags => ?TAGS, + security => [#{'bearerAuth' => []}], responses => #{ 200 => delete([api_secret], fields(app)) } @@ -47,6 +48,7 @@ schema("/api_key") -> post => #{ description => ?DESC(create_new_api_key), tags => ?TAGS, + security => [#{'bearerAuth' => []}], 'requestBody' => delete([created_at, api_key, api_secret], fields(app)), responses => #{ 200 => hoconsc:ref(app), diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 559344e2b..7745207ce 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -184,11 +184,11 @@ list() -> to_map(ets:match_object(?APP, #?APP{_ = '_'})). authorize(<<"/api/v5/users", _/binary>>, _Req, _ApiKey, _ApiSecret) -> - {error, <<"not_allowed">>}; + {error, <<"not_allowed">>, <<"users">>}; authorize(<<"/api/v5/api_key", _/binary>>, _Req, _ApiKey, _ApiSecret) -> - {error, <<"not_allowed">>}; + {error, <<"not_allowed">>, <<"api_key">>}; authorize(<<"/api/v5/logout", _/binary>>, _Req, _ApiKey, _ApiSecret) -> - {error, <<"not_allowed">>}; + {error, <<"not_allowed">>, <<"logout">>}; authorize(_Path, Req, ApiKey, ApiSecret) -> Now = erlang:system_time(second), case find_by_api_key(ApiKey) of diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index d437e07c9..760ab1732 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -394,8 +394,23 @@ t_authorize(_Config) -> {ok, _Status} = emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader), ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, KeyError)), ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, BanPath, SecretError)), - ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, ApiKeyPath, BasicHeader)), ?assertEqual(Unauthorized, emqx_mgmt_api_test_util:request_api(get, UserPath, BasicHeader)), + {error, {{"HTTP/1.1", 401, "Unauthorized"}, _Headers, Body}} = + emqx_mgmt_api_test_util:request_api( + get, + ApiKeyPath, + [], + BasicHeader, + [], + #{return_all => true} + ), + ?assertMatch( + #{ + <<"code">> := <<"API_KEY_NOT_ALLOW">>, + <<"message">> := _ + }, + emqx_utils_json:decode(Body, [return_maps]) + ), ?assertMatch( {ok, #{<<"api_key">> := _, <<"enable">> := false}}, diff --git a/rel/i18n/emqx_mgmt_api_api_keys.hocon b/rel/i18n/emqx_mgmt_api_api_keys.hocon index 8acbe60d0..4becd01aa 100644 --- a/rel/i18n/emqx_mgmt_api_api_keys.hocon +++ b/rel/i18n/emqx_mgmt_api_api_keys.hocon @@ -1,27 +1,27 @@ emqx_mgmt_api_api_keys { api_key_list.desc: -"""Return api_key list""" +"""Return api_key list. This API can only be requested using a bearer token.""" api_key_list.label: """Return api_key list""" create_new_api_key.desc: -"""Create new api_key""" +"""Create new api_key. This API can only be requested using a bearer token.""" create_new_api_key.label: """Create new api_key""" get_api_key.desc: -"""Return the specific api_key""" +"""Return the specific api_key. This API can only be requested using a bearer token.""" get_api_key.label: """Return the specific api_key""" update_api_key.desc: -"""Update the specific api_key""" +"""Update the specific api_key. This API can only be requested using a bearer token.""" update_api_key.label: """Update the specific api_key""" delete_api_key.desc: -"""Delete the specific api_key""" +"""Delete the specific api_key. This API can only be requested using a bearer token.""" delete_api_key.label: """Delete the specific api_key""" From 218af3fef4b62c3ca8b6c5ac57360c69585f84c6 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 23 Jan 2024 14:14:23 +0800 Subject: [PATCH 21/32] chore: update ecql to 0.6.0 --- apps/emqx_bridge_cassandra/rebar.config | 2 +- .../src/emqx_bridge_cassandra_connector.erl | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/apps/emqx_bridge_cassandra/rebar.config b/apps/emqx_bridge_cassandra/rebar.config index c0a72fef9..04ee603fa 100644 --- a/apps/emqx_bridge_cassandra/rebar.config +++ b/apps/emqx_bridge_cassandra/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.2"}}}, + {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.6.0"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index 3db71c9e0..3b30f1d26 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -159,7 +159,7 @@ on_add_channel(_InstId, #{channels := Channs} = OldState, ChannId, ChannConf0) - #{parameters := #{cql := CQL}} = ChannConf0, {PrepareCQL, ParamsTokens} = emqx_placeholder:preproc_sql(CQL, '?'), ParsedCql = #{ - prepare_key => short_prepare_key(ChannId), + prepare_key => make_prepare_key(ChannId), prepare_cql => PrepareCQL, params_tokens => ParamsTokens }, @@ -462,10 +462,5 @@ maybe_assign_type(V) when is_float(V) -> {double, V}; maybe_assign_type(V) -> V. -short_prepare_key(Str) when is_binary(Str) -> - true = size(Str) > 0, - Sha = crypto:hash(sha, Str), - %% TODO: change to binary:encode_hex(X, lowercase) when OTP version is always > 25 - Hex = string:lowercase(binary:encode_hex(Sha)), - <> = Hex, - binary_to_atom(<<"cassa_prepare_key:", UniqueEnough/binary>>). +make_prepare_key(ChannId) -> + ChannId. From 1eb47d0c16a3a592c5d3eb0693ff7536f31e7ba9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Jan 2024 18:18:17 -0300 Subject: [PATCH 22/32] perf(ds): inherit only LTS paths containing wildcards when adding a new generation Fixes https://github.com/emqx/emqx/pull/12338#discussion_r1462139499 --- apps/emqx_durable_storage/src/emqx_ds_lts.erl | 148 +++++++++++++++++- .../src/emqx_ds_storage_bitfield_lts.erl | 27 ++-- 2 files changed, 152 insertions(+), 23 deletions(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_lts.erl index 9d87cf571..226af62f0 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_lts.erl @@ -20,7 +20,7 @@ -export([ trie_create/1, trie_create/0, trie_restore/2, - trie_restore_existing/2, + trie_copy_learned_paths/2, topic_key/3, match_topics/2, lookup_topic_key/2 @@ -120,10 +120,6 @@ trie_create() -> -spec trie_restore(options(), [{_Key, _Val}]) -> trie(). trie_restore(Options, Dump) -> Trie = trie_create(Options), - trie_restore_existing(Trie, Dump). - --spec trie_restore_existing(trie(), [{_Key, _Val}]) -> trie(). -trie_restore_existing(Trie, Dump) -> lists:foreach( fun({{StateFrom, Token}, StateTo}) -> trie_insert(Trie, StateFrom, Token, StateTo) @@ -132,6 +128,17 @@ trie_restore_existing(Trie, Dump) -> ), Trie. +-spec trie_copy_learned_paths(trie(), trie()) -> trie(). +trie_copy_learned_paths(OldTrie, NewTrie) -> + WildcardPaths = [P || P <- paths(OldTrie), contains_wildcard(P)], + lists:foreach( + fun({{StateFrom, Token}, StateTo}) -> + trie_insert(NewTrie, StateFrom, Token, StateTo) + end, + lists:flatten(WildcardPaths) + ), + NewTrie. + %% @doc Lookup the topic key. Create a new one, if not found. -spec topic_key(trie(), threshold_fun(), [binary() | '']) -> msg_storage_key(). topic_key(Trie, ThresholdFun, Tokens) -> @@ -385,6 +392,41 @@ emanating(#trie{trie = Tab}, State, Token) when is_binary(Token); Token =:= '' - ets:lookup(Tab, {State, Token}) ]. +all_emanating(#trie{trie = Tab}, State) -> + ets:select( + Tab, + ets:fun2ms(fun(#trans{key = {S, Edge}, next = Next}) when S == State -> + {{S, Edge}, Next} + end) + ). + +paths(#trie{} = T) -> + Roots = all_emanating(T, ?PREFIX), + lists:flatmap( + fun({Segment, Next}) -> + follow_path(T, Next, [{Segment, Next}]) + end, + Roots + ). + +follow_path(#trie{} = T, State, Path) -> + lists:flatmap( + fun + ({{_State, ?EOT}, _Next} = Segment) -> + [lists:reverse([Segment | Path])]; + ({_Edge, Next} = Segment) -> + follow_path(T, Next, [Segment | Path]) + end, + all_emanating(T, State) + ). + +contains_wildcard([{{_State, ?PLUS}, _Next} | _Rest]) -> + true; +contains_wildcard([_ | Rest]) -> + contains_wildcard(Rest); +contains_wildcard([]) -> + false. + %%================================================================================ %% Tests %%================================================================================ @@ -636,4 +678,100 @@ test_key(Trie, Threshold, Topic0) -> {ok, Ret} = lookup_topic_key(Trie, Topic), Ret. +paths_test() -> + T = trie_create(), + Threshold = 4, + ThresholdFun = fun + (0) -> 1000; + (_) -> Threshold + end, + PathsToInsert = + [ + [''], + [1], + [2, 2], + [3, 3, 3], + [2, 3, 4] + ] ++ [[4, I, 4] || I <- lists:seq(1, Threshold + 2)] ++ + [['', I, ''] || I <- lists:seq(1, Threshold + 2)], + lists:foreach( + fun(PathSpec) -> + test_key(T, ThresholdFun, PathSpec) + end, + PathsToInsert + ), + + %% Test that the paths we've inserted are produced in the output + Paths = paths(T), + FormattedPaths = lists:map(fun format_path/1, Paths), + ExpectedWildcardPaths = + [ + [4, '+', 4], + ['', '+', ''] + ], + ExpectedPaths = + [ + [''], + [1], + [2, 2], + [3, 3, 3] + ] ++ [[4, I, 4] || I <- lists:seq(1, Threshold)] ++ + [['', I, ''] || I <- lists:seq(1, Threshold)] ++ + ExpectedWildcardPaths, + FormatPathSpec = + fun(PathSpec) -> + lists:map( + fun + (I) when is_integer(I) -> integer_to_binary(I); + (A) -> A + end, + PathSpec + ) ++ [?EOT] + end, + lists:foreach( + fun(PathSpec) -> + Path = FormatPathSpec(PathSpec), + ?assert( + lists:member(Path, FormattedPaths), + #{ + paths => FormattedPaths, + expected_path => Path + } + ) + end, + ExpectedPaths + ), + + %% Test filter function for paths containing wildcards + WildcardPaths = lists:filter(fun contains_wildcard/1, Paths), + FormattedWildcardPaths = lists:map(fun format_path/1, WildcardPaths), + ?assertEqual( + sets:from_list(FormattedWildcardPaths, [{version, 2}]), + sets:from_list(lists:map(FormatPathSpec, ExpectedWildcardPaths), [{version, 2}]), + #{ + expected => ExpectedWildcardPaths, + wildcards => FormattedWildcardPaths + } + ), + + %% Test that we're able to reconstruct the same trie from the paths + T2 = trie_create(), + [ + trie_insert(T2, State, Edge, Next) + || Path <- Paths, + {{State, Edge}, Next} <- Path + ], + #trie{trie = Tab1} = T, + #trie{trie = Tab2} = T2, + Dump1 = sets:from_list(ets:tab2list(Tab1), [{version, 2}]), + Dump2 = sets:from_list(ets:tab2list(Tab2), [{version, 2}]), + ?assertEqual(Dump1, Dump2), + + ok. + +format_path([{{_State, Edge}, _Next} | Rest]) -> + [Edge | format_path(Rest)]; +format_path([]) -> + []. + -endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl index 2a3086a57..d407dab41 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl @@ -205,17 +205,15 @@ open(_Shard, DBHandle, GenId, CFRefs, Schema) -> s(). post_creation_actions( #{ - db := DBHandle, - old_gen_id := OldGenId, - old_cf_refs := OldCFRefs, - new_gen_runtime_data := NewGenData0 + new_gen_runtime_data := NewGenData, + old_gen_runtime_data := OldGenData } ) -> - {_, OldTrieCF} = lists:keyfind(trie_cf(OldGenId), 1, OldCFRefs), - #s{trie = NewTrie0} = NewGenData0, - NewTrie = copy_previous_trie(DBHandle, NewTrie0, OldTrieCF), + #s{trie = OldTrie} = OldGenData, + #s{trie = NewTrie0} = NewGenData, + NewTrie = copy_previous_trie(OldTrie, NewTrie0), ?tp(bitfield_lts_inherited_trie, #{}), - NewGenData0#s{trie = NewTrie}. + NewGenData#s{trie = NewTrie}. -spec drop( emqx_ds_storage_layer:shard_id(), @@ -533,16 +531,9 @@ restore_trie(TopicIndexBytes, DB, CF) -> rocksdb:iterator_close(IT) end. --spec copy_previous_trie(rocksdb:db_handle(), emqx_ds_lts:trie(), rocksdb:cf_handle()) -> - emqx_ds_lts:trie(). -copy_previous_trie(DBHandle, NewTrie, OldCF) -> - {ok, IT} = rocksdb:iterator(DBHandle, OldCF, []), - try - OldDump = read_persisted_trie(IT, rocksdb:iterator_move(IT, first)), - emqx_ds_lts:trie_restore_existing(NewTrie, OldDump) - after - rocksdb:iterator_close(IT) - end. +-spec copy_previous_trie(emqx_ds_lts:trie(), emqx_ds_lts:trie()) -> emqx_ds_lts:trie(). +copy_previous_trie(OldTrie, NewTrie) -> + emqx_ds_lts:trie_copy_learned_paths(OldTrie, NewTrie). read_persisted_trie(IT, {ok, KeyB, ValB}) -> [ From eecd7e084c7747c2cb486c642b2f90af691284d0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 23 Jan 2024 09:47:03 -0300 Subject: [PATCH 23/32] test(ds): reduce flakiness --- .../test/emqx_persistent_messages_SUITE.erl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index c46d726f4..d0b939540 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -438,10 +438,19 @@ t_message_gc(Config) -> TopicFilter = emqx_topic:words(<<"#">>), StartTime = 0, Msgs = consume(TopicFilter, StartTime), - %% only "1" and "2" should have been GC'ed - ?assertEqual( - sets:from_list([<<"3">>, <<"4">>], [{version, 2}]), - sets:from_list([emqx_message:payload(Msg) || Msg <- Msgs], [{version, 2}]) + %% "1" and "2" should have been GC'ed + PresentMessages = sets:from_list( + [emqx_message:payload(Msg) || Msg <- Msgs], + [{version, 2}] + ), + ?assert( + sets:is_empty( + sets:intersection( + PresentMessages, + sets:from_list([<<"1">>, <<"2">>], [{version, 2}]) + ) + ), + #{present_messages => PresentMessages} ), ok From 4afba8eb94bfdce05f55c7f9d68a56b78024b233 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 24 Jan 2024 18:50:57 +0800 Subject: [PATCH 24/32] feat: port emqx/emqx-enterprise#1892, add some SQL functions --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 65 ++++++++-- .../test/emqx_rule_funcs_SUITE.erl | 111 ++++++++++++++++-- 2 files changed, 158 insertions(+), 18 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index aa5e271eb..da78ed875 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -116,7 +116,9 @@ %% Data Type Validation Funcs -export([ is_null/1, + is_null_var/1, is_not_null/1, + is_not_null_var/1, is_str/1, is_bool/1, is_int/1, @@ -153,6 +155,9 @@ ascii/1, find/2, find/3, + join_to_string/1, + join_to_string/2, + join_to_sql_values_string/1, jq/2, jq/3 ]). @@ -163,7 +168,10 @@ -export([ map_get/2, map_get/3, - map_put/3 + map_put/3, + map_keys/1, + map_values/1, + map_to_entries/1 ]). %% For backward compatibility @@ -699,9 +707,16 @@ hexstr2bin(Str) when is_binary(Str) -> is_null(undefined) -> true; is_null(_Data) -> false. +%% Similar to is_null/1, but also works for the JSON value 'null' +is_null_var(null) -> true; +is_null_var(Data) -> is_null(Data). + is_not_null(Data) -> not is_null(Data). +is_not_null_var(Data) -> + not is_null_var(Data). + is_str(T) when is_binary(T) -> true; is_str(_) -> false. @@ -847,6 +862,23 @@ find_s(S, P, Dir) -> SubStr -> SubStr end. +join_to_string(List) when is_list(List) -> + join_to_string(<<", ">>, List). +join_to_string(Sep, List) when is_list(List), is_binary(Sep) -> + iolist_to_binary(lists:join(Sep, [str(Item) || Item <- List])). +join_to_sql_values_string(List) -> + QuotedList = + [ + case is_list(Item) of + true -> + emqx_placeholder:quote_sql(emqx_utils_json:encode(Item)); + false -> + emqx_placeholder:quote_sql(Item) + end + || Item <- List + ], + iolist_to_binary(lists:join(<<", ">>, QuotedList)). + -spec jq(FilterProgram, JSON, TimeoutMS) -> Result when FilterProgram :: binary(), JSON :: binary() | term(), @@ -920,7 +952,8 @@ map_put(Key, Val, Map) -> mget(Key, Map) -> mget(Key, Map, undefined). -mget(Key, Map, Default) -> +mget(Key, Map0, Default) -> + Map = map(Map0), case maps:find(Key, Map) of {ok, Val} -> Val; @@ -947,7 +980,8 @@ mget(Key, Map, Default) -> Default end. -mput(Key, Val, Map) -> +mput(Key, Val, Map0) -> + Map = map(Map0), case maps:find(Key, Map) of {ok, _} -> maps:put(Key, Val, Map); @@ -974,6 +1008,13 @@ mput(Key, Val, Map) -> maps:put(Key, Val, Map) end. +map_keys(Map) -> + maps:keys(map(Map)). +map_values(Map) -> + maps:values(map(Map)). +map_to_entries(Map) -> + [#{key => K, value => V} || {K, V} <- maps:to_list(map(Map))]. + %%------------------------------------------------------------------------------ %% Hash Funcs %%------------------------------------------------------------------------------ @@ -1168,16 +1209,18 @@ map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. function_literal(Fun, []) when is_atom(Fun) -> - atom_to_list(Fun) ++ "()"; + iolist_to_binary(atom_to_list(Fun) ++ "()"); function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> WithFirstArg = io_lib:format("~ts(~0p", [atom_to_list(Fun), FArg]), - lists:foldl( - fun(Arg, Literal) -> - io_lib:format("~ts, ~0p", [Literal, Arg]) - end, - WithFirstArg, - Args - ) ++ ")"; + FuncLiteral = + lists:foldl( + fun(Arg, Literal) -> + io_lib:format("~ts, ~0p", [Literal, Arg]) + end, + WithFirstArg, + Args + ) ++ ")", + iolist_to_binary(FuncLiteral); function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 93b35fbc4..d157cee7b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -215,15 +215,32 @@ hex_convert() -> ). t_is_null(_) -> + ?assertEqual(false, emqx_rule_funcs:is_null(null)), ?assertEqual(true, emqx_rule_funcs:is_null(undefined)), + ?assertEqual(false, emqx_rule_funcs:is_null(<<"undefined">>)), ?assertEqual(false, emqx_rule_funcs:is_null(a)), ?assertEqual(false, emqx_rule_funcs:is_null(<<>>)), ?assertEqual(false, emqx_rule_funcs:is_null(<<"a">>)). +t_is_null_var(_) -> + ?assertEqual(true, emqx_rule_funcs:is_null_var(null)), + ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"null">>)), + ?assertEqual(true, emqx_rule_funcs:is_null_var(undefined)), + ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"undefined">>)), + ?assertEqual(false, emqx_rule_funcs:is_null_var(a)), + ?assertEqual(false, emqx_rule_funcs:is_null_var(<<>>)), + ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"a">>)). + t_is_not_null(_) -> [ ?assertEqual(emqx_rule_funcs:is_not_null(T), not emqx_rule_funcs:is_null(T)) - || T <- [undefined, a, <<"a">>, <<>>] + || T <- [undefined, <<"undefined">>, null, <<"null">>, a, <<"a">>, <<>>] + ]. + +t_is_not_null_var(_) -> + [ + ?assertEqual(emqx_rule_funcs:is_not_null_var(T), not emqx_rule_funcs:is_null_var(T)) + || T <- [undefined, <<"undefined">>, null, <<"null">>, a, <<"a">>, <<>>] ]. t_is_str(_) -> @@ -622,6 +639,63 @@ t_ascii(_) -> ?assertEqual(97, apply_func(ascii, [<<"a">>])), ?assertEqual(97, apply_func(ascii, [<<"ab">>])). +t_join_to_string(_) -> + A = 1, + B = a, + C = <<"c">>, + D = #{a => 1}, + E = [1, 2, 3], + F = [#{<<"key">> => 1, <<"value">> => 2}], + M = #{<<"a">> => a, <<"b">> => 1, <<"c">> => <<"c">>}, + J = <<"{\"a\":\"a\",\"b\":1,\"c\":\"c\"}">>, + ?assertEqual(<<"a,b,c">>, apply_func(join_to_string, [<<",">>, [<<"a">>, <<"b">>, <<"c">>]])), + ?assertEqual(<<"a b c">>, apply_func(join_to_string, [<<" ">>, [<<"a">>, <<"b">>, <<"c">>]])), + ?assertEqual( + <<"a, b, c">>, apply_func(join_to_string, [<<", ">>, [<<"a">>, <<"b">>, <<"c">>]]) + ), + ?assertEqual( + <<"1, a, c, {\"a\":1}, [1,2,3], [{\"value\":2,\"key\":1}]">>, + apply_func(join_to_string, [<<", ">>, [A, B, C, D, E, F]]) + ), + ?assertEqual(<<"a">>, apply_func(join_to_string, [<<",">>, [<<"a">>]])), + ?assertEqual(<<"">>, apply_func(join_to_string, [<<",">>, []])), + ?assertEqual(<<"a, b, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_keys(M)])), + ?assertEqual(<<"a, b, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_keys(J)])), + ?assertEqual(<<"a, 1, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_values(M)])), + ?assertEqual(<<"a, 1, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_values(J)])). + +t_join_to_sql_values_string(_) -> + A = 1, + B = a, + C = <<"c">>, + D = #{a => 1}, + E = [1, 2, 3], + E1 = [97, 98], + F = [#{<<"key">> => 1, <<"value">> => 2}], + M = #{<<"a">> => a, <<"b">> => 1, <<"c">> => <<"c">>}, + J = <<"{\"a\":\"a\",\"b\":1,\"c\":\"c\"}">>, + ?assertEqual( + <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [[<<"a">>, <<"b">>, <<"c">>]]) + ), + ?assertEqual( + <<"1, 'a', 'c', '{\"a\":1}', '[1,2,3]', '[97,98]', '[{\"value\":2,\"key\":1}]'">>, + apply_func(join_to_sql_values_string, [[A, B, C, D, E, E1, F]]) + ), + ?assertEqual(<<"'a'">>, apply_func(join_to_sql_values_string, [[<<"a">>]])), + ?assertEqual(<<"">>, apply_func(join_to_sql_values_string, [[]])), + ?assertEqual( + <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_keys(M)]) + ), + ?assertEqual( + <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_keys(J)]) + ), + ?assertEqual( + <<"'a', 1, 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_values(M)]) + ), + ?assertEqual( + <<"'a', 1, 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_values(J)]) + ). + t_find(_) -> ?assertEqual(<<"cbcd">>, apply_func(find, [<<"acbcd">>, <<"c">>])), ?assertEqual(<<"cbcd">>, apply_func(find, [<<"acbcd">>, <<"c">>, <<"leading">>])), @@ -746,14 +820,37 @@ t_map_put(_) -> ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])). t_mget(_) -> - ?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])), - ?assertEqual(1, apply_func(map_get, [<<"a">>, #{<<"a">> => 1}])), - ?assertEqual(undefined, apply_func(map_get, [<<"a">>, #{}])). + ?assertEqual(1, apply_func(mget, [<<"a">>, #{a => 1}])), + ?assertEqual(1, apply_func(mget, [<<"a">>, <<"{\"a\" : 1}">>])), + ?assertEqual(1, apply_func(mget, [<<"a">>, #{<<"a">> => 1}])), + ?assertEqual(1, apply_func(mget, [<<"a.b">>, #{<<"a.b">> => 1}])), + ?assertEqual(undefined, apply_func(mget, [<<"a">>, #{}])). t_mput(_) -> - ?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])), - ?assertEqual(#{<<"a">> => 2}, apply_func(map_put, [<<"a">>, 2, #{<<"a">> => 1}])), - ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])). + ?assertEqual(#{<<"a">> => 1}, apply_func(mput, [<<"a">>, 1, #{}])), + ?assertEqual(#{<<"a">> => 2}, apply_func(mput, [<<"a">>, 2, #{<<"a">> => 1}])), + ?assertEqual(#{<<"a">> => 2}, apply_func(mput, [<<"a">>, 2, <<"{\"a\" : 1}">>])), + ?assertEqual(#{<<"a.b">> => 2}, apply_func(mput, [<<"a.b">>, 2, #{<<"a.b">> => 1}])), + ?assertEqual(#{a => 2}, apply_func(mput, [<<"a">>, 2, #{a => 1}])). + +t_map_to_entries(_) -> + ?assertEqual([], apply_func(map_to_entries, [#{}])), + M = #{a => 1, b => <<"b">>}, + J = <<"{\"a\":1,\"b\":\"b\"}">>, + ?assertEqual( + [ + #{key => a, value => 1}, + #{key => b, value => <<"b">>} + ], + apply_func(map_to_entries, [M]) + ), + ?assertEqual( + [ + #{key => <<"a">>, value => 1}, + #{key => <<"b">>, value => <<"b">>} + ], + apply_func(map_to_entries, [J]) + ). t_bitsize(_) -> ?assertEqual(8, apply_func(bitsize, [<<"a">>])), From d840561036b74ddbc1361d65a4f1789af414559e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 24 Jan 2024 21:53:23 +0800 Subject: [PATCH 25/32] build: direct paths to avoid wildcard traversal of the _build dir - erlfmt always try loop through files in the `--exclude-files` dir --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 48ca7ebcb..739195926 100644 --- a/Makefile +++ b/Makefile @@ -316,10 +316,9 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) .PHONY: fmt fmt: $(REBAR) @$(SCRIPTS)/erlfmt -w 'apps/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}' - @$(SCRIPTS)/erlfmt -w '**/*.escript' --exclude-files '_build/**' - @$(SCRIPTS)/erlfmt -w '**/rebar.config' --exclude-files '_build/**' - @$(SCRIPTS)/erlfmt -w 'rebar.config.erl' - @$(SCRIPTS)/erlfmt -w 'bin/nodetool' + @$(SCRIPTS)/erlfmt -w 'apps/*/rebar.config' 'apps/emqx/rebar.config.script' '.ci/fvt_tests/http_server/rebar.config' + @$(SCRIPTS)/erlfmt -w 'rebar.config' 'rebar.config.erl' + @$(SCRIPTS)/erlfmt -w 'scripts/*.escript' 'bin/*.escript' 'bin/nodetool' @mix format .PHONY: clean-test-cluster-config From 5547a40ceb95d9be2537a173df2f44900b87ce2a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jan 2024 11:39:14 -0300 Subject: [PATCH 26/32] fix(ds): don't use env var for data dir default value Fixes https://github.com/emqx/emqx/pull/12380 --- apps/emqx/src/emqx_persistent_message.erl | 4 ++-- apps/emqx/src/emqx_schema.erl | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_message.erl b/apps/emqx/src/emqx_persistent_message.erl index d725c9b2c..8e3755fdb 100644 --- a/apps/emqx/src/emqx_persistent_message.erl +++ b/apps/emqx/src/emqx_persistent_message.erl @@ -61,13 +61,13 @@ force_ds() -> emqx_config:get([session_persistence, force_persistence]). storage_backend(#{ - builtin := #{ + builtin := Opts = #{ enable := true, - data_dir := DataDir, n_shards := NShards, replication_factor := ReplicationFactor } }) -> + DataDir = maps:get(data_dir, Opts, emqx:data_dir()), #{ backend => builtin, data_dir => DataDir, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 56d575bd9..bbca13172 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1896,11 +1896,8 @@ fields("session_storage_backend_builtin") -> string(), #{ desc => ?DESC(session_builtin_data_dir), - default => <<"${EMQX_DATA_DIR}">>, - importance => ?IMPORTANCE_LOW, - converter => fun(Path, Opts) -> - naive_env_interpolation(ensure_unicode_path(Path, Opts)) - end + required => false, + importance => ?IMPORTANCE_LOW } )}, {"n_shards", From 9b7df302e889e876e09047272c7e36b2fcc202d6 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:42:24 +0100 Subject: [PATCH 27/32] fix(ds): Cache database metadata in RAM --- .../src/emqx_ds_replication_layer_meta.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl index 38c2dbbe7..16c52f20e 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl @@ -388,7 +388,7 @@ ensure_tables() -> {rlog_shard, ?SHARD}, {majority, Majority}, {type, ordered_set}, - {storage, rocksdb_copies}, + {storage, disc_copies}, {record_name, ?META_TAB}, {attributes, record_info(fields, ?META_TAB)} ]), @@ -396,7 +396,7 @@ ensure_tables() -> {rlog_shard, ?SHARD}, {majority, Majority}, {type, ordered_set}, - {storage, rocksdb_copies}, + {storage, disc_copies}, {record_name, ?NODE_TAB}, {attributes, record_info(fields, ?NODE_TAB)} ]), From 137535a82140313b1b3334e75717862cb91bac48 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:45:56 +0100 Subject: [PATCH 28/32] feat(ds): Introduce egress process for the builtin backend --- .../emqx_persistent_session_ds_SUITE.erl | 11 +- apps/emqx/src/emqx_persistent_message.erl | 4 +- .../test/emqx_persistent_messages_SUITE.erl | 4 +- .../test/emqx_persistent_session_SUITE.erl | 2 +- apps/emqx_durable_storage/src/emqx_ds.erl | 8 +- apps/emqx_durable_storage/src/emqx_ds_app.erl | 2 +- .../src/emqx_ds_builtin_db_sup.erl | 152 +++++++++++++++ .../src/emqx_ds_builtin_sup.erl | 132 +++++++++++++ .../src/emqx_ds_replication_layer.erl | 112 ++++++----- .../src/emqx_ds_replication_layer.hrl | 33 ++++ .../src/emqx_ds_replication_layer_egress.erl | 175 ++++++++++++++++++ .../src/emqx_ds_replication_layer_meta.erl | 12 +- .../src/emqx_ds_storage_layer.erl | 6 +- .../src/emqx_ds_storage_layer_sup.erl | 4 +- apps/emqx_durable_storage/src/emqx_ds_sup.erl | 66 +++---- .../test/emqx_ds_SUITE.erl | 2 +- .../emqx_ds_storage_bitfield_lts_SUITE.erl | 31 ++-- 17 files changed, 615 insertions(+), 141 deletions(-) create mode 100644 apps/emqx_durable_storage/src/emqx_ds_builtin_db_sup.erl create mode 100644 apps/emqx_durable_storage/src/emqx_ds_builtin_sup.erl create mode 100644 apps/emqx_durable_storage/src/emqx_ds_replication_layer.hrl create mode 100644 apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl diff --git a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl index b5beb9ae7..fba36601f 100644 --- a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_persistent_session_ds_SUITE). @@ -18,6 +18,9 @@ %% CT boilerplate %%------------------------------------------------------------------------------ +suite() -> + [{timetrap, {seconds, 60}}]. + all() -> emqx_common_test_helpers:all(?MODULE). @@ -191,6 +194,7 @@ t_non_persistent_session_subscription(_Config) -> ClientId = atom_to_binary(?FUNCTION_NAME), SubTopicFilter = <<"t/#">>, ?check_trace( + #{timetrap => 30_000}, begin ?tp(notice, "starting", #{}), Client = start_client(#{ @@ -220,6 +224,7 @@ t_session_subscription_idempotency(Config) -> SubTopicFilter = <<"t/+">>, ClientId = <<"myclientid">>, ?check_trace( + #{timetrap => 30_000}, begin ?force_ordering( #{?snk_kind := persistent_session_ds_subscription_added}, @@ -281,6 +286,7 @@ t_session_unsubscription_idempotency(Config) -> SubTopicFilter = <<"t/+">>, ClientId = <<"myclientid">>, ?check_trace( + #{timetrap => 30_000}, begin ?force_ordering( #{ @@ -385,6 +391,7 @@ do_t_session_discard(Params) -> ReconnectOpts = ReconnectOpts0#{clientid => ClientId}, SubTopicFilter = <<"t/+">>, ?check_trace( + #{timetrap => 30_000}, begin ?tp(notice, "starting", #{}), Client0 = start_client(#{ @@ -472,6 +479,7 @@ do_t_session_expiration(_Config, Opts) -> } = Opts, CommonParams = #{proto_ver => v5, clientid => ClientId}, ?check_trace( + #{timetrap => 30_000}, begin Topic = <<"some/topic">>, Params0 = maps:merge(CommonParams, FirstConn), @@ -539,6 +547,7 @@ t_session_gc(Config) -> end, ?check_trace( + #{timetrap => 30_000}, begin ClientId0 = <<"session_gc0">>, Client0 = StartClient(ClientId0, Port1, 30), diff --git a/apps/emqx/src/emqx_persistent_message.erl b/apps/emqx/src/emqx_persistent_message.erl index 8e3755fdb..effad17dd 100644 --- a/apps/emqx/src/emqx_persistent_message.erl +++ b/apps/emqx/src/emqx_persistent_message.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -99,7 +99,7 @@ needs_persistence(Msg) -> -spec store_message(emqx_types:message()) -> emqx_ds:store_batch_result(). store_message(Msg) -> - emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg]). + emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg], #{sync => false}). has_subscribers(#message{topic = Topic}) -> emqx_persistent_session_ds_router:has_any_route(Topic). diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index 36c8848cf..0c0eaac28 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -376,7 +376,7 @@ t_publish_empty_topic_levels(_Config) -> {<<"t/3/bar">>, <<"6">>} ], [emqtt:publish(Pub, Topic, Payload, ?QOS_1) || {Topic, Payload} <- Messages], - Received = receive_messages(length(Messages), 1_500), + Received = receive_messages(length(Messages)), ?assertMatch( [ #{topic := <<"t//1/">>, payload := <<"2">>}, diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index 09cbf306d..f2a42332e 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 434169520..1402a19e3 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -111,11 +111,15 @@ %% use in emqx_guid. Otherwise, the iterators won't match the message timestamps. -type time() :: non_neg_integer(). --type message_store_opts() :: #{}. +-type message_store_opts() :: + #{ + sync => boolean() + }. -type generic_db_opts() :: #{ backend := atom(), + serialize_by => clientid | topic, _ => _ }. diff --git a/apps/emqx_durable_storage/src/emqx_ds_app.erl b/apps/emqx_durable_storage/src/emqx_ds_app.erl index 858855b6f..dcf353a99 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_app.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_app.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ds_app). diff --git a/apps/emqx_durable_storage/src/emqx_ds_builtin_db_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_builtin_db_sup.erl new file mode 100644 index 000000000..9df07eb18 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_builtin_db_sup.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% @doc Supervisor that contains all the processes that belong to a +%% given builtin DS database. +-module(emqx_ds_builtin_db_sup). + +-behaviour(supervisor). + +%% API: +-export([start_db/2, start_shard/1, start_egress/1, stop_shard/1, ensure_shard/1]). + +%% behaviour callbacks: +-export([init/1]). + +%% internal exports: +-export([start_link_sup/2]). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +-define(via(REC), {via, gproc, {n, l, REC}}). + +-define(db_sup, ?MODULE). +-define(shard_sup, emqx_ds_builtin_db_shard_sup). +-define(egress_sup, emqx_ds_builtin_db_egress_sup). + +-record(?db_sup, {db}). +-record(?shard_sup, {db}). +-record(?egress_sup, {db}). + +%%================================================================================ +%% API funcions +%%================================================================================ + +-spec start_db(emqx_ds:db(), emqx_ds_replication_layer:builtin_db_opts()) -> {ok, pid()}. +start_db(DB, Opts) -> + start_link_sup(#?db_sup{db = DB}, Opts). + +-spec start_shard(emqx_ds_storage_layer:shard_id()) -> + supervisor:startchild_ret(). +start_shard(Shard = {DB, _}) -> + supervisor:start_child(?via(#?shard_sup{db = DB}), shard_spec(DB, Shard)). + +-spec start_egress(emqx_ds_storage_layer:shard_id()) -> + supervisor:startchild_ret(). +start_egress({DB, Shard}) -> + supervisor:start_child(?via(#?egress_sup{db = DB}), egress_spec(DB, Shard)). + +-spec stop_shard(emqx_ds_storage_layer:shard_id()) -> ok | {error, _}. +stop_shard(Shard = {DB, _}) -> + Sup = ?via(#?shard_sup{db = DB}), + ok = supervisor:terminate_child(Sup, Shard), + ok = supervisor:delete_child(Sup, Shard). + +-spec ensure_shard(emqx_ds_storage_layer:shard_id()) -> + ok | {error, _Reason}. +ensure_shard(Shard) -> + case start_shard(Shard) of + {ok, _Pid} -> + ok; + {error, {already_started, _Pid}} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +%%================================================================================ +%% behaviour callbacks +%%================================================================================ + +init({#?db_sup{db = DB}, DefaultOpts}) -> + %% Spec for the top-level supervisor for the database: + logger:notice("Starting DS DB ~p", [DB]), + _ = emqx_ds_replication_layer_meta:open_db(DB, DefaultOpts), + %% TODO: before the leader election is implemented, we set ourselves as the leader for all shards: + MyShards = emqx_ds_replication_layer_meta:my_shards(DB), + lists:foreach( + fun(Shard) -> + emqx_ds_replication_layer:maybe_set_myself_as_leader(DB, Shard) + end, + MyShards + ), + Children = [sup_spec(#?shard_sup{db = DB}, []), sup_spec(#?egress_sup{db = DB}, [])], + SupFlags = #{ + strategy => one_for_all, + intensity => 0, + period => 1 + }, + {ok, {SupFlags, Children}}; +init({#?shard_sup{db = DB}, _}) -> + %% Spec for the supervisor that manages the worker processes for + %% each local shard of the DB: + MyShards = emqx_ds_replication_layer_meta:my_shards(DB), + Children = [shard_spec(DB, Shard) || Shard <- MyShards], + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1 + }, + {ok, {SupFlags, Children}}; +init({#?egress_sup{db = DB}, _}) -> + %% Spec for the supervisor that manages the egress proxy processes + %% managing traffic towards each of the shards of the DB: + Shards = emqx_ds_replication_layer_meta:shards(DB), + Children = [egress_spec(DB, Shard) || Shard <- Shards], + SupFlags = #{ + strategy => one_for_one, + intensity => 0, + period => 1 + }, + {ok, {SupFlags, Children}}. + +%%================================================================================ +%% Internal exports +%%================================================================================ + +start_link_sup(Id, Options) -> + supervisor:start_link(?via(Id), ?MODULE, {Id, Options}). + +%%================================================================================ +%% Internal functions +%%================================================================================ + +sup_spec(Id, Options) -> + #{ + id => element(1, Id), + start => {?MODULE, start_link_sup, [Id, Options]}, + type => supervisor, + shutdown => infinity + }. + +shard_spec(DB, Shard) -> + Options = emqx_ds_replication_layer_meta:get_options(DB), + #{ + id => Shard, + start => {emqx_ds_storage_layer, start_link, [{DB, Shard}, Options]}, + shutdown => 5_000, + restart => permanent, + type => worker + }. + +egress_spec(DB, Shard) -> + #{ + id => Shard, + start => {emqx_ds_replication_layer_egress, start_link, [DB, Shard]}, + shutdown => 5_000, + restart => permanent, + type => worker + }. diff --git a/apps/emqx_durable_storage/src/emqx_ds_builtin_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_builtin_sup.erl new file mode 100644 index 000000000..50ed18de1 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_builtin_sup.erl @@ -0,0 +1,132 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This supervisor manages the global worker processes needed for +%% the functioning of builtin databases, and all builtin database +%% attach to it. +-module(emqx_ds_builtin_sup). + +-behaviour(supervisor). + +%% API: +-export([start_db/2, stop_db/1]). + +%% behavior callbacks: +-export([init/1]). + +%% internal exports: +-export([start_top/0, start_databases_sup/0]). + +-export_type([]). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +-define(top, ?MODULE). +-define(databases, emqx_ds_builtin_databases_sup). + +%%================================================================================ +%% API functions +%%================================================================================ + +-spec start_db(emqx_ds:db(), emqx_ds_replication_layer:builtin_db_opts()) -> + supervisor:startchild_ret(). +start_db(DB, Opts) -> + ensure_top(), + ChildSpec = #{ + id => DB, + start => {emqx_ds_builtin_db_sup, start_db, [DB, Opts]}, + type => supervisor, + shutdown => infinity + }, + supervisor:start_child(?databases, ChildSpec). + +-spec stop_db(emqx_ds:db()) -> ok. +stop_db(DB) -> + case whereis(?databases) of + Pid when is_pid(Pid) -> + _ = supervisor:terminate_child(?databases, DB), + _ = supervisor:delete_child(?databases, DB), + ok; + undefined -> + ok + end. + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +%% There are two layers of supervision: +%% +%% 1. top supervisor for the builtin backend. It contains the global +%% worker processes (like the metadata server), and `?databases' +%% supervisior. +%% +%% 2. `?databases': a `one_for_one' supervisor where each child is a +%% `db' supervisor that contains processes that represent the DB. +%% Chidren are attached dynamically to this one. +init(?top) -> + %% Children: + MetadataServer = #{ + id => metadata_server, + start => {emqx_ds_replication_layer_meta, start_link, []}, + restart => permanent, + type => worker, + shutdown => 5000 + }, + DBsSup = #{ + id => ?databases, + start => {?MODULE, start_databases_sup, []}, + restart => permanent, + type => supervisor, + shutdown => infinity + }, + %% + SupFlags = #{ + strategy => one_for_all, + intensity => 1, + period => 1, + auto_shutdown => never + }, + {ok, {SupFlags, [MetadataServer, DBsSup]}}; +init(?databases) -> + %% Children are added dynamically: + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 1 + }, + {ok, {SupFlags, []}}. + +%%================================================================================ +%% Internal exports +%%================================================================================ + +-spec start_top() -> {ok, pid()}. +start_top() -> + supervisor:start_link({local, ?top}, ?MODULE, ?top). + +start_databases_sup() -> + supervisor:start_link({local, ?databases}, ?MODULE, ?databases). + +%%================================================================================ +%% Internal functions +%%================================================================================ + +ensure_top() -> + {ok, _} = emqx_ds_sup:attach_backend(builtin, {?MODULE, start_top, []}), + ok. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl index 387587570..7f696e3ce 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl @@ -32,7 +32,10 @@ get_streams/3, make_iterator/4, update_iterator/3, - next/3 + next/3, + node_of_shard/2, + shard_of_message/3, + maybe_set_myself_as_leader/2 ]). %% internal exports: @@ -51,24 +54,12 @@ -export_type([shard_id/0, builtin_db_opts/0, stream/0, iterator/0, message_id/0, batch/0]). -include_lib("emqx_utils/include/emqx_message.hrl"). +-include("emqx_ds_replication_layer.hrl"). %%================================================================================ %% Type declarations %%================================================================================ -%% # "Record" integer keys. We use maps with integer keys to avoid persisting and sending -%% records over the wire. - -%% tags: --define(STREAM, 1). --define(IT, 2). --define(BATCH, 3). - -%% keys: --define(tag, 1). --define(shard, 2). --define(enc, 3). - -type shard_id() :: binary(). -type builtin_db_opts() :: @@ -101,8 +92,6 @@ -type message_id() :: emqx_ds:message_id(). --define(batch_messages, 2). - -type batch() :: #{ ?tag := ?BATCH, ?batch_messages := [emqx_types:message()] @@ -120,16 +109,14 @@ list_shards(DB) -> -spec open_db(emqx_ds:db(), builtin_db_opts()) -> ok | {error, _}. open_db(DB, CreateOpts) -> - ok = emqx_ds_sup:ensure_workers(), - Opts = emqx_ds_replication_layer_meta:open_db(DB, CreateOpts), - MyShards = emqx_ds_replication_layer_meta:my_shards(DB), - lists:foreach( - fun(Shard) -> - emqx_ds_storage_layer:open_shard({DB, Shard}, Opts), - maybe_set_myself_as_leader(DB, Shard) - end, - MyShards - ). + case emqx_ds_builtin_sup:start_db(DB, CreateOpts) of + {ok, _} -> + ok; + {error, {already_started, _}} -> + ok; + {error, Err} -> + {error, Err} + end. -spec add_generation(emqx_ds:db()) -> ok | {error, _}. add_generation(DB) -> @@ -170,17 +157,15 @@ drop_generation(DB, {Shard, GenId}) -> -spec drop_db(emqx_ds:db()) -> ok | {error, _}. drop_db(DB) -> Nodes = list_nodes(), - _ = emqx_ds_proto_v1:drop_db(Nodes, DB), + _ = emqx_ds_proto_v2:drop_db(Nodes, DB), _ = emqx_ds_replication_layer_meta:drop_db(DB), + emqx_ds_builtin_sup:stop_db(DB), ok. -spec store_batch(emqx_ds:db(), [emqx_types:message(), ...], emqx_ds:message_store_opts()) -> emqx_ds:store_batch_result(). store_batch(DB, Messages, Opts) -> - Shard = shard_of_messages(DB, Messages), - Node = node_of_shard(DB, Shard), - Batch = #{?tag => ?BATCH, ?batch_messages => Messages}, - emqx_ds_proto_v1:store_batch(Node, DB, Shard, Batch, Opts). + emqx_ds_replication_layer_egress:store_batch(DB, Messages, Opts). -spec get_streams(emqx_ds:db(), emqx_ds:topic_filter(), emqx_ds:time()) -> [{emqx_ds:stream_rank(), stream()}]. @@ -262,6 +247,41 @@ next(DB, Iter0, BatchSize) -> Other end. +-spec node_of_shard(emqx_ds:db(), shard_id()) -> node(). +node_of_shard(DB, Shard) -> + case emqx_ds_replication_layer_meta:shard_leader(DB, Shard) of + {ok, Leader} -> + Leader; + {error, no_leader_for_shard} -> + %% TODO: use optvar + timer:sleep(500), + node_of_shard(DB, Shard) + end. + +-spec shard_of_message(emqx_ds:db(), emqx_types:message(), clientid | topic) -> + emqx_ds_replication_layer:shard_id(). +shard_of_message(DB, #message{from = From, topic = Topic}, SerializeBy) -> + N = emqx_ds_replication_layer_meta:n_shards(DB), + Hash = + case SerializeBy of + clientid -> erlang:phash2(From, N); + topic -> erlang:phash2(Topic, N) + end, + integer_to_binary(Hash). + +%% TODO: there's no real leader election right now +-spec maybe_set_myself_as_leader(emqx_ds:db(), shard_id()) -> ok. +maybe_set_myself_as_leader(DB, Shard) -> + Site = emqx_ds_replication_layer_meta:this_site(), + case emqx_ds_replication_layer_meta:in_sync_replicas(DB, Shard) of + [Site | _] -> + %% Currently the first in-sync replica always becomes the + %% leader + ok = emqx_ds_replication_layer_meta:set_leader(DB, Shard, node()); + _Sites -> + ok + end. + %%================================================================================ %% behavior callbacks %%================================================================================ @@ -273,6 +293,7 @@ next(DB, Iter0, BatchSize) -> -spec do_drop_db_v1(emqx_ds:db()) -> ok | {error, _}. do_drop_db_v1(DB) -> MyShards = emqx_ds_replication_layer_meta:my_shards(DB), + emqx_ds_builtin_sup:stop_db(DB), lists:foreach( fun(Shard) -> emqx_ds_storage_layer:drop_shard({DB, Shard}) @@ -354,34 +375,5 @@ do_drop_generation_v3(DB, ShardId, GenId) -> %% Internal functions %%================================================================================ -%% TODO: there's no real leader election right now --spec maybe_set_myself_as_leader(emqx_ds:db(), shard_id()) -> ok. -maybe_set_myself_as_leader(DB, Shard) -> - Site = emqx_ds_replication_layer_meta:this_site(), - case emqx_ds_replication_layer_meta:in_sync_replicas(DB, Shard) of - [Site | _] -> - %% Currently the first in-sync replica always becomes the - %% leader - ok = emqx_ds_replication_layer_meta:set_leader(DB, Shard, node()); - _Sites -> - ok - end. - --spec node_of_shard(emqx_ds:db(), shard_id()) -> node(). -node_of_shard(DB, Shard) -> - case emqx_ds_replication_layer_meta:shard_leader(DB, Shard) of - {ok, Leader} -> - Leader; - {error, no_leader_for_shard} -> - %% TODO: use optvar - timer:sleep(500), - node_of_shard(DB, Shard) - end. - -%% Here we assume that all messages in the batch come from the same client -shard_of_messages(DB, [#message{from = From} | _]) -> - N = emqx_ds_replication_layer_meta:n_shards(DB), - integer_to_binary(erlang:phash2(From, N)). - list_nodes() -> mria:running_nodes(). diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.hrl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.hrl new file mode 100644 index 000000000..42e72f258 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.hrl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022, 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_DS_REPLICATION_LAYER_HRL). +-define(EMQX_DS_REPLICATION_LAYER_HRL, true). + +%% # "Record" integer keys. We use maps with integer keys to avoid persisting and sending +%% records over the wire. + +%% tags: +-define(STREAM, 1). +-define(IT, 2). +-define(BATCH, 3). + +%% keys: +-define(tag, 1). +-define(shard, 2). +-define(enc, 3). +-define(batch_messages, 2). + +-endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl new file mode 100644 index 000000000..3b264d9d1 --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl @@ -0,0 +1,175 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Egress servers are responsible for proxing the outcoming +%% `store_batch' requests towards EMQX DS shards. +%% +%% They re-assemble messages from different local processes into +%% fixed-sized batches, and introduce centralized channels between the +%% nodes. They are also responsible for maintaining backpressure +%% towards the local publishers. +%% +%% There is (currently) one egress process for each shard running on +%% each node, but it should be possible to have a pool of egress +%% servers, if needed. +-module(emqx_ds_replication_layer_egress). + +-behaviour(gen_server). + +%% API: +-export([start_link/2, store_batch/3]). + +%% behavior callbacks: +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +%% internal exports: +-export([]). + +-export_type([]). + +-include("emqx_ds_replication_layer.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). + +%%================================================================================ +%% Type declarations +%%================================================================================ + +-define(via(DB, Shard), {via, gproc, {n, l, {?MODULE, DB, Shard}}}). +-define(flush, flush). + +-record(enqueue_req, {message :: emqx_types:message(), sync :: boolean()}). + +%%================================================================================ +%% API functions +%%================================================================================ + +-spec start_link(emqx_ds:db(), emqx_ds_replication_layer:shard_id()) -> {ok, pid()}. +start_link(DB, Shard) -> + gen_server:start_link(?via(DB, Shard), ?MODULE, [DB, Shard], []). + +-spec store_batch(emqx_ds:db(), [emqx_types:message()], emqx_ds:message_store_opts()) -> + ok. +store_batch(DB, Messages, Opts) -> + Sync = maps:get(sync, Opts, true), + lists:foreach( + fun(Message) -> + Shard = emqx_ds_replication_layer:shard_of_message(DB, Message, clientid), + gen_server:call(?via(DB, Shard), #enqueue_req{message = Message, sync = Sync}) + end, + Messages + ). + +%%================================================================================ +%% behavior callbacks +%%================================================================================ + +-record(s, { + db :: emqx_ds:db(), + shard :: emqx_ds_replication_layer:shard_id(), + leader :: node(), + n = 0 :: non_neg_integer(), + tref :: reference(), + batch = [] :: [emqx_types:message()], + pending_replies = [] :: [gen_server:from()] +}). + +init([DB, Shard]) -> + process_flag(trap_exit, true), + process_flag(message_queue_data, off_heap), + %% TODO: adjust leader dynamically + {ok, Leader} = emqx_ds_replication_layer_meta:shard_leader(DB, Shard), + S = #s{ + db = DB, + shard = Shard, + leader = Leader, + tref = start_timer() + }, + {ok, S}. + +handle_call(#enqueue_req{message = Msg, sync = Sync}, From, S) -> + do_enqueue(From, Sync, Msg, S); +handle_call(_Call, _From, S) -> + {reply, {error, unknown_call}, S}. + +handle_cast(_Cast, S) -> + {noreply, S}. + +handle_info(?flush, S) -> + {noreply, do_flush(S)}; +handle_info(_Info, S) -> + {noreply, S}. + +terminate(_Reason, _S) -> + ok. + +%%================================================================================ +%% Internal exports +%%================================================================================ + +%%================================================================================ +%% Internal functions +%%================================================================================ + +do_flush(S = #s{batch = []}) -> + S#s{tref = start_timer()}; +do_flush( + S = #s{batch = Messages, pending_replies = Replies, db = DB, shard = Shard, leader = Leader} +) -> + Batch = #{?tag => ?BATCH, ?batch_messages => lists:reverse(Messages)}, + ok = emqx_ds_proto_v2:store_batch(Leader, DB, Shard, Batch, #{}), + [gen_server:reply(From, ok) || From <- lists:reverse(Replies)], + ?tp(emqx_ds_replication_layer_egress_flush, #{db => DB, shard => Shard}), + erlang:garbage_collect(), + S#s{ + n = 0, + batch = [], + pending_replies = [], + tref = start_timer() + }. + +do_enqueue(From, Sync, Msg, S0 = #s{n = N, batch = Batch, pending_replies = Replies}) -> + NMax = 1000, + S1 = S0#s{n = N + 1, batch = [Msg | Batch]}, + S2 = + case N >= NMax of + true -> + _ = erlang:cancel_timer(S0#s.tref), + do_flush(S1); + false -> + S1 + end, + %% TODO: later we may want to delay the reply until the message is + %% replicated, but it requies changes to the PUBACK/PUBREC flow to + %% allow for async replies. For now, we ack when the message is + %% _buffered_ rather than stored. + %% + %% Otherwise, the client would freeze for at least flush interval, + %% or until the buffer is filled. + S = + case Sync of + true -> + S2#s{pending_replies = [From | Replies]}; + false -> + gen_server:reply(From, ok), + S2 + end, + %% TODO: add a backpressure mechanism for the server to avoid + %% building a long message queue. + {noreply, S}. + +start_timer() -> + Interval = 10, + erlang:send_after(Interval, self(), flush). diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl index 16c52f20e..b49b0e8f7 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ in_sync_replicas/2, sites/0, open_db/2, + get_options/1, update_db_config/2, drop_db/1, shard_leader/2, @@ -230,6 +231,11 @@ is_leader(Node) -> {atomic, Result} = mria:transaction(?SHARD, fun ?MODULE:is_leader_trans/1, [Node]), Result. +-spec get_options(emqx_ds:db()) -> emqx_ds_replication_layer:builtin_db_opts(). +get_options(DB) -> + {atomic, Opts} = mria:transaction(?SHARD, fun ?MODULE:open_db_trans/2, [DB, undefined]), + Opts. + -spec open_db(emqx_ds:db(), emqx_ds_replication_layer:builtin_db_opts()) -> emqx_ds_replication_layer:builtin_db_opts(). open_db(DB, DefaultOpts) -> @@ -293,11 +299,11 @@ terminate(_Reason, #s{}) -> %% Internal exports %%================================================================================ --spec open_db_trans(emqx_ds:db(), emqx_ds_replication_layer:builtin_db_opts()) -> +-spec open_db_trans(emqx_ds:db(), emqx_ds_replication_layer:builtin_db_opts() | undefined) -> emqx_ds_replication_layer:builtin_db_opts(). open_db_trans(DB, CreateOpts) -> case mnesia:wread({?META_TAB, DB}) of - [] -> + [] when is_map(CreateOpts) -> NShards = maps:get(n_shards, CreateOpts), ReplicationFactor = maps:get(replication_factor, CreateOpts), mnesia:write(#?META_TAB{db = DB, db_props = CreateOpts}), diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl index 0dcb8ce52..85a94a846 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -199,7 +199,6 @@ open_shard(Shard, Options) -> -spec drop_shard(shard_id()) -> ok. drop_shard(Shard) -> - catch emqx_ds_storage_layer_sup:stop_shard(Shard), case persistent_term:get({?MODULE, Shard, data_dir}, undefined) of undefined -> ok; @@ -586,7 +585,8 @@ commit_metadata(#s{shard_id = ShardId, schema = Schema, shard = Runtime, db = DB rocksdb_open(Shard, Options) -> DBOptions = [ {create_if_missing, true}, - {create_missing_column_families, true} + {create_missing_column_families, true}, + {enable_write_thread_adaptive_yield, false} | maps:get(db_options, Options, []) ], DataDir = maps:get(data_dir, Options, emqx:data_dir()), diff --git a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl index fd8cf289f..424b35133 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_storage_layer_sup.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ds_storage_layer_sup). @@ -23,7 +23,7 @@ -spec start_link() -> {ok, pid()}. start_link() -> - supervisor:start_link({local, ?SUP}, ?MODULE, []). + supervisor:start_link(?MODULE, []). -spec start_shard(emqx_ds_storage_layer:shard_id(), emqx_ds:create_db_opts()) -> supervisor:startchild_ret(). diff --git a/apps/emqx_durable_storage/src/emqx_ds_sup.erl b/apps/emqx_durable_storage/src/emqx_ds_sup.erl index 819d7d874..e863e74ce 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_sup.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_sup.erl @@ -6,7 +6,7 @@ -behaviour(supervisor). %% API: --export([start_link/0, ensure_workers/0]). +-export([start_link/0, attach_backend/2]). %% behaviour callbacks: -export([init/1]). @@ -25,64 +25,40 @@ start_link() -> supervisor:start_link({local, ?SUP}, ?MODULE, top). --spec ensure_workers() -> ok. -ensure_workers() -> - ChildSpec = #{ - id => workers_sup, - restart => temporary, - type => supervisor, - start => {supervisor, start_link, [?MODULE, workers]} +%% @doc Attach a child backend-specific supervisor to the top +%% application supervisor, if not yet present +-spec attach_backend(_BackendId, {module(), atom(), list()}) -> + {ok, pid()} | {error, _}. +attach_backend(Backend, Start) -> + Spec = #{ + id => Backend, + start => Start, + significant => false, + shutdown => infinity, + type => supervisor }, - case supervisor:start_child(?SUP, ChildSpec) of - {ok, _} -> - ok; - {error, already_present} -> - ok; - {error, {already_started, _}} -> - ok + case supervisor:start_child(?SUP, Spec) of + {ok, Pid} -> + {ok, Pid}; + {error, {already_started, Pid}} -> + {ok, Pid}; + {error, Err} -> + {error, Err} end. %%================================================================================ %% behaviour callbacks %%================================================================================ --dialyzer({nowarn_function, init/1}). init(top) -> + Children = [], SupFlags = #{ - strategy => one_for_all, + strategy => one_for_one, intensity => 10, period => 1 }, - {ok, {SupFlags, []}}; -init(workers) -> - %% TODO: technically, we don't need rocksDB for the alternative - %% backends. But right now we have any: - Children = [meta(), storage_layer_sup()], - SupFlags = #{ - strategy => one_for_all, - intensity => 0, - period => 1 - }, {ok, {SupFlags, Children}}. %%================================================================================ %% Internal functions %%================================================================================ - -meta() -> - #{ - id => emqx_ds_replication_layer_meta, - start => {emqx_ds_replication_layer_meta, start_link, []}, - restart => permanent, - type => worker, - shutdown => 5000 - }. - -storage_layer_sup() -> - #{ - id => local_store_shard_sup, - start => {emqx_ds_storage_layer_sup, start_link, []}, - restart => permanent, - type => supervisor, - shutdown => infinity - }. diff --git a/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl index d7dccccf5..9dae8e699 100644 --- a/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl +++ b/apps/emqx_durable_storage/test/emqx_ds_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl index 5d32143a7..03d86dd88 100644 --- a/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl +++ b/apps/emqx_durable_storage/test/emqx_ds_storage_bitfield_lts_SUITE.erl @@ -16,8 +16,8 @@ -define(DEFAULT_CONFIG, #{ backend => builtin, storage => {emqx_ds_storage_bitfield_lts, #{}}, - n_shards => 16, - replication_factor => 3 + n_shards => 1, + replication_factor => 1 }). -define(COMPACT_CONFIG, #{ @@ -26,15 +26,10 @@ {emqx_ds_storage_bitfield_lts, #{ bits_per_wildcard_level => 8 }}, - n_shards => 16, - replication_factor => 3 + n_shards => 1, + replication_factor => 1 }). -%% Smoke test for opening and reopening the database -t_open(_Config) -> - ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), - {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}). - %% Smoke test of store function t_store(_Config) -> MessageID = emqx_guid:gen(), @@ -98,8 +93,8 @@ t_get_streams(_Config) -> [FooBarBaz] = GetStream(<<"foo/bar/baz">>), [A] = GetStream(<<"a">>), %% Restart shard to make sure trie is persisted and restored: - ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), - {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}), + ok = emqx_ds_builtin_sup:stop_db(?FUNCTION_NAME), + {ok, _} = emqx_ds_builtin_sup:start_db(?FUNCTION_NAME, #{}), %% Verify that there are no "ghost streams" for topics that don't %% have any messages: [] = GetStream(<<"bar/foo">>), @@ -196,9 +191,9 @@ t_replay(_Config) -> ?assert(check(?SHARD, <<"foo/+/+">>, 0, Messages)), ?assert(check(?SHARD, <<"+/+/+">>, 0, Messages)), ?assert(check(?SHARD, <<"+/+/baz">>, 0, Messages)), - %% Restart shard to make sure trie is persisted and restored: - ok = emqx_ds_storage_layer_sup:stop_shard(?SHARD), - {ok, _} = emqx_ds_storage_layer_sup:start_shard(?SHARD, #{}), + %% Restart the DB to make sure trie is persisted and restored: + ok = emqx_ds_builtin_sup:stop_db(?FUNCTION_NAME), + {ok, _} = emqx_ds_builtin_sup:start_db(?FUNCTION_NAME, #{}), %% Learned wildcard topics: ?assertNot(check(?SHARD, <<"wildcard/1000/suffix/foo">>, 0, [])), ?assert(check(?SHARD, <<"wildcard/1/suffix/foo">>, 0, Messages)), @@ -412,21 +407,21 @@ suite() -> [{timetrap, {seconds, 20}}]. init_per_suite(Config) -> {ok, _} = application:ensure_all_started(emqx_durable_storage), - emqx_ds_sup:ensure_workers(), Config. end_per_suite(_Config) -> ok = application:stop(emqx_durable_storage). init_per_testcase(TC, Config) -> - {ok, _} = emqx_ds_storage_layer_sup:start_shard(shard(TC), ?DEFAULT_CONFIG), + ok = emqx_ds:open_db(TC, ?DEFAULT_CONFIG), Config. end_per_testcase(TC, _Config) -> - ok = emqx_ds_storage_layer_sup:stop_shard(shard(TC)). + emqx_ds:drop_db(TC), + ok. shard(TC) -> - {?MODULE, atom_to_binary(TC)}. + {TC, <<"0">>}. keyspace(TC) -> TC. From eee221f1d018f2fef7a528f3ba16a20fd78784f9 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 23 Jan 2024 21:29:37 +0100 Subject: [PATCH 29/32] feat(ds): Make egress batching configurable --- apps/emqx/src/emqx_schema.erl | 20 ++++++++++++++++++- .../src/emqx_ds_replication_layer_egress.erl | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bbca13172..b03cfe72e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1915,6 +1915,24 @@ fields("session_storage_backend_builtin") -> default => 3, importance => ?IMPORTANCE_HIDDEN } + )}, + {"egress_batch_size", + sc( + pos_integer(), + #{ + default => 1000, + mapping => "emqx_durable_storage.egress_batch_size", + importance => ?IMPORTANCE_HIDDEN + } + )}, + {"egress_flush_interval", + sc( + timeout_duration_ms(), + #{ + default => 100, + mapping => "emqx_durable_storage.egress_flush_interval", + importance => ?IMPORTANCE_HIDDEN + } )} ]. diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl index 3b264d9d1..842e8e5ed 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl @@ -141,7 +141,7 @@ do_flush( }. do_enqueue(From, Sync, Msg, S0 = #s{n = N, batch = Batch, pending_replies = Replies}) -> - NMax = 1000, + NMax = application:get_env(emqx_durable_storage, egress_batch_size, 1000), S1 = S0#s{n = N + 1, batch = [Msg | Batch]}, S2 = case N >= NMax of @@ -171,5 +171,5 @@ do_enqueue(From, Sync, Msg, S0 = #s{n = N, batch = Batch, pending_replies = Repl {noreply, S}. start_timer() -> - Interval = 10, + Interval = application:get_env(emqx_durable_storage, egress_flush_interval, 100), erlang:send_after(Interval, self(), flush). From 305a54f646d7539fa6820a721430130486735e87 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 24 Jan 2024 14:43:09 +0100 Subject: [PATCH 30/32] chore(ds): Update BPAPI version --- .../src/emqx_ds_replication_layer.erl | 12 ++++++------ .../src/emqx_ds_replication_layer_egress.erl | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl index 7f696e3ce..7432fe3c7 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer.erl @@ -121,7 +121,7 @@ open_db(DB, CreateOpts) -> -spec add_generation(emqx_ds:db()) -> ok | {error, _}. add_generation(DB) -> Nodes = emqx_ds_replication_layer_meta:leader_nodes(DB), - _ = emqx_ds_proto_v2:add_generation(Nodes, DB), + _ = emqx_ds_proto_v3:add_generation(Nodes, DB), ok. -spec update_db_config(emqx_ds:db(), builtin_db_opts()) -> ok | {error, _}. @@ -157,7 +157,7 @@ drop_generation(DB, {Shard, GenId}) -> -spec drop_db(emqx_ds:db()) -> ok | {error, _}. drop_db(DB) -> Nodes = list_nodes(), - _ = emqx_ds_proto_v2:drop_db(Nodes, DB), + _ = emqx_ds_proto_v3:drop_db(Nodes, DB), _ = emqx_ds_replication_layer_meta:drop_db(DB), emqx_ds_builtin_sup:stop_db(DB), ok. @@ -174,7 +174,7 @@ get_streams(DB, TopicFilter, StartTime) -> lists:flatmap( fun(Shard) -> Node = node_of_shard(DB, Shard), - Streams = emqx_ds_proto_v1:get_streams(Node, DB, Shard, TopicFilter, StartTime), + Streams = emqx_ds_proto_v3:get_streams(Node, DB, Shard, TopicFilter, StartTime), lists:map( fun({RankY, Stream}) -> RankX = Shard, @@ -196,7 +196,7 @@ get_streams(DB, TopicFilter, StartTime) -> make_iterator(DB, Stream, TopicFilter, StartTime) -> #{?tag := ?STREAM, ?shard := Shard, ?enc := StorageStream} = Stream, Node = node_of_shard(DB, Shard), - case emqx_ds_proto_v1:make_iterator(Node, DB, Shard, StorageStream, TopicFilter, StartTime) of + case emqx_ds_proto_v3:make_iterator(Node, DB, Shard, StorageStream, TopicFilter, StartTime) of {ok, Iter} -> {ok, #{?tag => ?IT, ?shard => Shard, ?enc => Iter}}; Err = {error, _} -> @@ -213,7 +213,7 @@ update_iterator(DB, OldIter, DSKey) -> #{?tag := ?IT, ?shard := Shard, ?enc := StorageIter} = OldIter, Node = node_of_shard(DB, Shard), case - emqx_ds_proto_v2:update_iterator( + emqx_ds_proto_v3:update_iterator( Node, DB, Shard, @@ -239,7 +239,7 @@ next(DB, Iter0, BatchSize) -> %% %% This kind of trickery should be probably done here in the %% replication layer. Or, perhaps, in the logic layer. - case emqx_ds_proto_v1:next(Node, DB, Shard, StorageIter0, BatchSize) of + case emqx_ds_proto_v3:next(Node, DB, Shard, StorageIter0, BatchSize) of {ok, StorageIter, Batch} -> Iter = Iter0#{?enc := StorageIter}, {ok, Iter, Batch}; diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl index 842e8e5ed..8b37b29cb 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_egress.erl @@ -172,4 +172,4 @@ do_enqueue(From, Sync, Msg, S0 = #s{n = N, batch = Batch, pending_replies = Repl start_timer() -> Interval = application:get_env(emqx_durable_storage, egress_flush_interval, 100), - erlang:send_after(Interval, self(), flush). + erlang:send_after(Interval, self(), ?flush). From 7b5f2948fe127c85dd70b3df693769b315251cbe Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 24 Jan 2024 18:38:15 +0100 Subject: [PATCH 31/32] test(ds): Fix flaky testcase --- apps/emqx/test/emqx_persistent_messages_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index 0c0eaac28..8a63d46be 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -475,7 +475,7 @@ t_metrics_not_dropped(_Config) -> {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Sub, <<"t/+">>, ?QOS_1), emqtt:publish(Pub, <<"t/ps">>, <<"payload">>, ?QOS_1), - ?assertMatch([_], receive_messages(1, 1_500)), + ?assertMatch([_], receive_messages(1)), DroppedAfter = emqx_metrics:val('messages.dropped'), DroppedNoSubAfter = emqx_metrics:val('messages.dropped.no_subscribers'), From 33981661c112d0ad11a96a28351cc266c9bc3c61 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 25 Jan 2024 09:20:23 +0800 Subject: [PATCH 32/32] chore: add changelogs for #12381 --- changes/ee/feat-12381.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ee/feat-12381.en.md diff --git a/changes/ee/feat-12381.en.md b/changes/ee/feat-12381.en.md new file mode 100644 index 000000000..dfd48db00 --- /dev/null +++ b/changes/ee/feat-12381.en.md @@ -0,0 +1,3 @@ +Added new SQL functions: map_keys(), map_values(), map_to_entries(), join_to_string(), join_to_string(), join_to_sql_values_string(), is_null_var(), is_not_null_var(). + +For more information on the functions and their usage, refer to the documentation.