From 5cabc6b3681e32aec077882e7bcdf529f6f93304 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 12 Jan 2024 18:21:31 +0800 Subject: [PATCH 01/62] chore: upgrade dashboard to e1.5.0-beta.3 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 020dd0192..48ca7ebcb 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.6.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.4.1 +export EMQX_EE_DASHBOARD_VERSION ?= e1.5.0-beta.3 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise From 61417f26d43e25b3f541c89bef56e9008e9fbf5e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Jan 2024 13:54:35 +0100 Subject: [PATCH 02/62] chore: bump versions to 5.5.x --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 0115edc91..6d9a1c22b 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.4.1"). +-define(EMQX_RELEASE_CE, "5.5.0"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.4.1"). +-define(EMQX_RELEASE_EE, "5.5.0-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 5f4dc4cdc..4e05e137b 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.1 +version: 5.5.1-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.1 +appVersion: 5.5.1-alpha.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index c95573c0a..41f0b37b6 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.1 +version: 5.5.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.1 +appVersion: 5.5.0 From 34681ec7a2a961e9ee7e9598f2bbf52a5c096aea Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 12 Jan 2024 14:05:29 +0100 Subject: [PATCH 03/62] chore: update scripts and ci from old release-xx branch to release-55 --- .github/workflows/build_packages_cron.yaml | 2 +- .github/workflows/codeql.yaml | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 +-- scripts/rel/cut.sh | 33 ++++------------------ scripts/rel/sync-remotes.sh | 22 ++++----------- 5 files changed, 15 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 784454c86..56d5c37f2 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -24,7 +24,7 @@ jobs: matrix: profile: - ['emqx', 'master', '5.3-2:1.15.7-26.2.1-2'] - - ['emqx-enterprise', 'release-54', '5.3-2:1.15.7-25.3.2-2'] + - ['emqx-enterprise', 'release-55', '5.3-2:1.15.7-25.3.2-2'] os: - debian10 - ubuntu22.04 diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index fd982d506..a51ef0d58 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -24,7 +24,7 @@ jobs: matrix: branch: - master - - release-54 + - release-55 language: - cpp - python diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 4e05e137b..59d92aa02 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.5.1-alpha.1 +version: 5.5.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.5.1-alpha.1 +appVersion: 5.5.0-alpha.1 diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index e218f8d0b..136f8c466 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -20,10 +20,7 @@ options: -h|--help: Print this usage. -b|--base: Specify the current release base branch, can be one of - release-51 - release-52 - release-53 - release-54 + release-55 NOTE: this option should be used when --dryrun. --dryrun: Do not actually create the git tag. @@ -38,7 +35,7 @@ options: For 5.X series the current working branch must be 'release-5X' --.--[ master ]---------------------------.-----------.--- \\ / - \`---[release-54]----(v5.4.0 | e5.4.0) + \`---[release-5X]----(v5.4.0 | e5.4.0) EOF } @@ -119,29 +116,11 @@ done rel_branch() { local tag="$1" case "$tag" in - v5.1.*) - echo 'release-51' + v5.5.*) + echo 'release-55' ;; - e5.1.*) - echo 'release-51' - ;; - v5.2.*) - echo 'release-52' - ;; - e5.2.*) - echo 'release-52' - ;; - v5.3.*) - echo 'release-53' - ;; - e5.3.*) - echo 'release-53' - ;; - v5.4.*) - echo 'release-54' - ;; - e5.4.*) - echo 'release-54' + e5.5.*) + echo 'release-55' ;; *) logerr "Unsupported version tag $TAG" diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index 6b41415f0..eb17995c2 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -5,7 +5,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." -BASE_BRANCHES=( 'release-54' 'release-53' 'release-52' 'release-51' 'master' ) +BASE_BRANCHES=( 'release-55' 'master' ) usage() { cat < Date: Mon, 8 Jan 2024 13:40:07 +0800 Subject: [PATCH 04/62] feat(cluster): expose the timeout parameter to invite node --- .../src/emqx_mgmt_api_cluster.erl | 38 ++++++-- .../src/proto/emqx_mgmt_cluster_proto_v3.erl | 38 ++++++++ .../test/emqx_mgmt_api_cluster_SUITE.erl | 94 ++++++++++++++++++- 3 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index be1b8e354..0a9a17ea2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -31,6 +31,8 @@ connected_replicants/0 ]). +-define(DEFAULT_INVITE_TIMEOUT, 15000). + namespace() -> "cluster". api_spec() -> @@ -77,6 +79,7 @@ schema("/cluster/:node/invite") -> desc => ?DESC(invite_node), tags => [<<"Cluster">>], parameters => [hoconsc:ref(node)], + 'requestBody' => hoconsc:ref(timeout), responses => #{ 200 => <<"ok">>, 400 => emqx_dashboard_swagger:error_codes(['BAD_REQUEST']) @@ -131,6 +134,14 @@ fields(core_replicants) -> #{desc => <<"Core node name">>, example => <<"emqx-core@127.0.0.1">>} )}, {replicant_nodes, ?HOCON(?ARRAY(?REF(replicant_info)))} + ]; +fields(timeout) -> + [ + {timeout, + ?HOCON( + non_neg_integer(), + #{desc => <<"Timeout in milliseconds">>, example => <<"15000">>} + )} ]. validate_node(Node) -> @@ -188,17 +199,24 @@ running_cores() -> Running = emqx:running_nodes(), lists:filter(fun(C) -> lists:member(C, Running) end, emqx:cluster_nodes(cores)). -invite_node(put, #{bindings := #{node := Node0}}) -> +invite_node(put, #{bindings := #{node := Node0}, body := Body}) -> Node = ekka_node:parse_name(binary_to_list(Node0)), - case emqx_mgmt_cluster_proto_v1:invite_node(Node, node()) of - ok -> - {200}; - ignore -> - {400, #{code => 'BAD_REQUEST', message => <<"Can't invite self">>}}; - {badrpc, Error} -> - {400, #{code => 'BAD_REQUEST', message => error_message(Error)}}; - {error, Error} -> - {400, #{code => 'BAD_REQUEST', message => error_message(Error)}} + case maps:get(<<"timeout">>, Body, ?DEFAULT_INVITE_TIMEOUT) of + T when not is_integer(T) -> + {400, #{code => 'BAD_REQUEST', message => <<"timeout must be integer">>}}; + T when T < 5000 -> + {400, #{code => 'BAD_REQUEST', message => <<"timeout can't less than 5000ms">>}}; + Timeout -> + case emqx_mgmt_cluster_proto_v3:invite_node(Node, node(), Timeout) of + ok -> + {200}; + ignore -> + {400, #{code => 'BAD_REQUEST', message => <<"Can't invite self">>}}; + {badrpc, Error} -> + {400, #{code => 'BAD_REQUEST', message => error_message(Error)}}; + {error, Error} -> + {400, #{code => 'BAD_REQUEST', message => error_message(Error)}} + end end. force_leave(delete, #{bindings := #{node := Node0}}) -> diff --git a/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl new file mode 100644 index 000000000..8110ac2cb --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl @@ -0,0 +1,38 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_cluster_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + invite_node/3, + connected_replicants/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.5.0". + +-spec invite_node(node(), node(), timeout()) -> ok | ignore | {error, term()} | emqx_rpc:badrpc(). +invite_node(Node, Self, Timeout) when is_integer(Timeout) -> + rpc:call(Node, emqx_mgmt_api_cluster, join, [Self], Timeout). + +-spec connected_replicants([node()]) -> emqx_rpc:multicall_result(). +connected_replicants(Nodes) -> + rpc:multicall(Nodes, emqx_mgmt_api_cluster, connected_replicants, [], 30_000). diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index 6e33c4001..3d4124b28 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -35,6 +35,9 @@ end_per_suite(_) -> init_per_testcase(TC = t_cluster_topology_api_replicants, Config0) -> Config = [{tc_name, TC} | Config0], [{cluster, cluster(Config)} | setup(Config)]; +init_per_testcase(TC = t_cluster_invite_api_timeout, Config0) -> + Config = [{tc_name, TC} | Config0], + [{cluster, cluster(Config)} | setup(Config)]; init_per_testcase(_TC, Config) -> emqx_mgmt_api_test_util:init_suite(?APPS), Config. @@ -42,6 +45,9 @@ init_per_testcase(_TC, Config) -> end_per_testcase(t_cluster_topology_api_replicants, Config) -> emqx_cth_cluster:stop(?config(cluster, Config)), cleanup(Config); +end_per_testcase(t_cluster_invite_api_timeout, Config) -> + emqx_cth_cluster:stop(?config(cluster, Config)), + cleanup(Config); end_per_testcase(_TC, _Config) -> emqx_mgmt_api_test_util:end_suite(?APPS). @@ -77,12 +83,94 @@ t_cluster_topology_api_replicants(Config) -> || Resp <- [lists:sort(R) || R <- [Core1Resp, Core2Resp, ReplResp]] ]. +t_cluster_invite_api_timeout(Config) -> + %% assert the cluster is created + [Core1, Core2, Replicant] = _NodesList = ?config(cluster, Config), + {200, Core1Resp} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := + [#{node := Replicant, streams := _}] + }, + #{ + core_node := Core2, + replicant_nodes := + [#{node := Replicant, streams := _}] + } + ], + lists:sort(Core1Resp) + ), + + %% force leave the core2 + {204} = rpc:call( + Core1, + emqx_mgmt_api_cluster, + force_leave, + [delete, #{bindings => #{node => atom_to_binary(Core2)}}] + ), + + %% assert the cluster is updated + {200, Core1Resp2} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := + [#{node := Replicant, streams := _}] + } + ], + lists:sort(Core1Resp2) + ), + + %% assert timeout parameter checking + Invite = fun(Node, Timeout) -> + Node1 = atom_to_binary(Node), + rpc:call( + Core1, + emqx_mgmt_api_cluster, + invite_node, + [put, #{bindings => #{node => Node1}, body => #{<<"timeout">> => Timeout}}] + ) + end, + ?assertMatch( + {400, #{code := 'BAD_REQUEST', message := <<"timeout must be integer">>}}, + Invite(Core2, not_a_integer_timeout) + ), + ?assertMatch( + {400, #{code := 'BAD_REQUEST', message := <<"timeout can't less than 5000ms">>}}, + Invite(Core2, 3000) + ), + + %% assert cluster is updated after invite + ?assertMatch( + {200}, + Invite(Core2, 15000) + ), + {200, Core1Resp3} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := + [#{node := Replicant, streams := _}] + }, + #{ + core_node := Core2, + replicant_nodes := _ + } + ], + lists:sort(Core1Resp3) + ). + cluster(Config) -> + NodeSpec = #{apps => ?APPS}, Nodes = emqx_cth_cluster:start( [ - {data_backup_core1, #{role => core, apps => ?APPS}}, - {data_backup_core2, #{role => core, apps => ?APPS}}, - {data_backup_replicant, #{role => replicant, apps => ?APPS}} + {data_backup_core1, NodeSpec#{role => core}}, + {data_backup_core2, NodeSpec#{role => core}}, + {data_backup_replicant, NodeSpec#{role => replicant}} ], #{work_dir => work_dir(Config)} ), From daad1521d42b35b3490e182fffee9eefbf2f0c38 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 8 Jan 2024 14:04:17 +0800 Subject: [PATCH 05/62] chore: update changes --- changes/ce/feat-12267.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/feat-12267.en.md diff --git a/changes/ce/feat-12267.en.md b/changes/ce/feat-12267.en.md new file mode 100644 index 000000000..70d3aec22 --- /dev/null +++ b/changes/ce/feat-12267.en.md @@ -0,0 +1,2 @@ +Add a new `timeout` parameter to the `cluster/:node/invite` interface. +Previously the default timeout was 5s which would often be caused by HTTP API calls due to emqx taking too long to join cluster. From 6ff4c560e430934acb74870214ef8959cf7dcd1e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Jan 2024 13:26:39 +0800 Subject: [PATCH 06/62] feat: support invite node in async mananer --- .../src/emqx_mgmt_api_cluster.erl | 127 ++++++++++++ .../emqx_management/src/emqx_mgmt_cluster.erl | 196 ++++++++++++++++++ apps/emqx_management/src/emqx_mgmt_sup.erl | 3 +- .../src/proto/emqx_mgmt_cluster_proto_v3.erl | 2 +- .../test/emqx_mgmt_api_cluster_SUITE.erl | 134 +++++++++++- rel/i18n/emqx_mgmt_api_cluster.hocon | 5 + 6 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_management/src/emqx_mgmt_cluster.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index 0a9a17ea2..1a46c0b36 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -26,6 +26,8 @@ cluster_info/2, cluster_topology/2, invite_node/2, + invite_node_async/2, + get_invitation_view/2, force_leave/2, join/1, connected_replicants/0 @@ -42,7 +44,9 @@ paths() -> [ "/cluster", "/cluster/topology", + "/cluster/invitation", "/cluster/:node/invite", + "/cluster/:node/invite_async", "/cluster/:node/force_leave" ]. @@ -72,6 +76,20 @@ schema("/cluster/topology") -> } } }; +schema("/cluster/invitation") -> + #{ + 'operationId' => get_invitation_view, + get => #{ + desc => ?DESC(get_invitation_view), + tags => [<<"Cluster">>], + responses => #{ + 200 => ?HOCON( + ?REF(invitation_view), + #{desc => <<"Get invitation progress created by async operation">>} + ) + } + } + }; schema("/cluster/:node/invite") -> #{ 'operationId' => invite_node, @@ -86,6 +104,19 @@ schema("/cluster/:node/invite") -> } } }; +schema("/cluster/:node/invite_async") -> + #{ + 'operationId' => invite_node_async, + put => #{ + desc => ?DESC(invite_node_async), + tags => [<<"Cluster">>], + parameters => [hoconsc:ref(node)], + responses => #{ + 200 => <<"ok">>, + 400 => emqx_dashboard_swagger:error_codes(['BAD_REQUEST']) + } + } + }; schema("/cluster/:node/force_leave") -> #{ 'operationId' => force_leave, @@ -142,6 +173,61 @@ fields(timeout) -> non_neg_integer(), #{desc => <<"Timeout in milliseconds">>, example => <<"15000">>} )} + ]; +fields(invitation_view) -> + [ + {succeed, + ?HOCON( + ?ARRAY(?REF(node_invitation_succeed)), + #{desc => <<"A list of information about nodes that were successfully invited">>} + )}, + {in_progress, + ?HOCON( + ?ARRAY(?REF(node_invitation_in_progress)), + #{desc => <<"A list of information about nodes that are processing invitations">>} + )}, + {failed, + ?HOCON( + ?ARRAY(?REF(node_invitation_failed)), + #{desc => <<"A list of information about nodes that failed to be invited">>} + )} + ]; +fields(node_invitation_failed) -> + fields(node_invitation_succeed) ++ + [ + {reason, + ?HOCON( + binary(), + #{desc => <<"Failed reason">>, example => <<"Bad RPC to target node">>} + )} + ]; +fields(node_invitation_succeed) -> + fields(node_invitation_in_progress) ++ + [ + {finished_at, + ?HOCON( + emqx_utils_calendar:epoch_millisecond(), + #{ + desc => <<"The end time of the invitation task, in millisecond">>, + example => <<"1705044829915">> + } + )} + ]; +fields(node_invitation_in_progress) -> + [ + {node, + ?HOCON( + binary(), + #{desc => <<"Node name">>, example => <<"emqx2@127.0.0.1">>} + )}, + {started_at, + ?HOCON( + emqx_utils_calendar:epoch_millisecond(), + #{ + desc => <<"The start time of the invitation task, in millisecond">>, + example => <<"1705044829915">> + } + )} ]. validate_node(Node) -> @@ -219,6 +305,23 @@ invite_node(put, #{bindings := #{node := Node0}, body := Body}) -> end end. +invite_node_async(put, #{bindings := #{node := Node0}}) -> + Node = ekka_node:parse_name(binary_to_list(Node0)), + case emqx_mgmt_cluster:invite_async(Node) of + ok -> + {200}; + ignore -> + {400, #{code => 'BAD_REQUEST', message => <<"Can't invite self">>}}; + {error, {already_started, _Pid}} -> + {400, #{ + code => 'BAD_REQUEST', + message => <<"The invitation task already created for this node">> + }} + end. + +get_invitation_view(get, _) -> + {200, format_invitation_view(emqx_mgmt_cluster:invitation_view())}. + force_leave(delete, #{bindings := #{node := Node0}}) -> Node = ekka_node:parse_name(binary_to_list(Node0)), case ekka:force_leave(Node) of @@ -240,3 +343,27 @@ connected_replicants() -> error_message(Msg) -> iolist_to_binary(io_lib:format("~p", [Msg])). + +format_invitation_view(#{ + succeed := Succeed, + in_progress := InProgress, + failed := Failed +}) -> + #{ + succeed => format_invitation_info(Succeed), + in_progress => format_invitation_info(InProgress), + failed => format_invitation_info(Failed) + }. + +format_invitation_info(L) when is_list(L) -> + lists:map( + fun(Info) -> + Info1 = emqx_utils_maps:update_if_present( + started_at, fun emqx_utils_calendar:epoch_to_rfc3339/1, Info + ), + emqx_utils_maps:update_if_present( + finished_at, fun emqx_utils_calendar:epoch_to_rfc3339/1, Info1 + ) + end, + L + ). diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl new file mode 100644 index 000000000..b5dfaae93 --- /dev/null +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mgmt_cluster). + +-behaviour(gen_server). + +%% APIs +-export([start_link/0]). + +-export([invite_async/1, invitation_view/0]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec invite_async(atom()) -> ok | ignore | {badrpc, any()}. +invite_async(Node) -> + JoinTo = node(), + case Node =/= JoinTo of + true -> + gen_server:call(?MODULE, {invite_async, Node, JoinTo}); + false -> + ignore + end. + +-spec invitation_view() -> map(). +invitation_view() -> + gen_server:call(?MODULE, invitation_view). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + process_flag(trap_exit, true), + {ok, #{}}. + +handle_call({invite_async, Node, JoinTo}, _From, State) -> + case maps:get(Node, State, undefined) of + undefined -> + Caller = self(), + Task = spawn_link_invite_worker(Node, JoinTo, Caller), + {reply, ok, State#{Node => Task}}; + WorkerPid -> + {reply, {error, {already_started, WorkerPid}}, State} + end; +handle_call(invitation_view, _From, State) -> + {reply, state_to_invitation_view(State), State}; +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({task_done, _WorkerPid, Node, Result}, State) -> + case maps:take(Node, State) of + {Task, State1} -> + History = maps:get(history, State1, #{}), + Task1 = Task#{ + result => Result, + finished_at => erlang:system_time(millisecond) + }, + {noreply, State1#{history => History#{Node => Task1}}}; + error -> + {noreply, State} + end; +handle_info({'EXIT', WorkerPid, Reason}, State) -> + case take_node_name_via_worker_pid(WorkerPid, State) of + {key_value, Node, Task, State1} -> + History = maps:get(history, State1, #{}), + Task1 = Task#{ + result => {error, Reason}, + finished_at => erlang:system_time(millisecond) + }, + {noreply, State1#{history => History#{Node => Task1}}}; + error -> + {noreply, State} + end; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +spawn_link_invite_worker(Node, JoinTo, Caller) -> + Pid = erlang:spawn_link( + fun() -> + Result = + case emqx_mgmt_cluster_proto_v3:invite_node(Node, JoinTo, infinity) of + ok -> + ok; + {error, {already_in_cluster, _Node}} -> + ok; + {error, _} = E -> + E; + {badrpc, Reason} -> + {error, {badrpc, Reason}} + end, + Caller ! {task_done, self(), Node, Result} + end + ), + #{worker => Pid, started_at => erlang:system_time(millisecond)}. + +take_node_name_via_worker_pid(WorkerPid, Map) when is_map(Map) -> + Key = find_node_name_via_worker_pid(WorkerPid, maps:next(maps:iterator(Map))), + case maps:take(Key, Map) of + error -> + error; + {Vaule, Map1} -> + {key_value, Key, Vaule, Map1} + end. + +find_node_name_via_worker_pid(_WorkerPid, none) -> + error; +find_node_name_via_worker_pid(WorkerPid, {Key, Task, I}) -> + case maps:get(worker, Task, undefined) of + WorkerPid -> + Key; + _ -> + find_node_name_via_worker_pid(WorkerPid, maps:next(I)) + end. + +state_to_invitation_view(State) -> + History = maps:get(history, State, #{}), + {Succ, Failed} = lists:foldl( + fun({Node, Task}, {SuccAcc, FailedAcc}) -> + #{ + started_at := StartedAt, + finished_at := FinishedAt, + result := Result + } = Task, + Ret = #{node => Node, started_at => StartedAt, finished_at => FinishedAt}, + case is_succeed_result(Result) of + true -> + {[Ret | SuccAcc], FailedAcc}; + false -> + {SuccAcc, [Ret#{reason => Result} | FailedAcc]} + end + end, + {[], []}, + maps:to_list(History) + ), + + InPro = maps:fold( + fun(Node, _Task = #{started_at := StartedAt}, Acc) -> + [#{node => Node, started_at => StartedAt} | Acc] + end, + [], + maps:without([history], State) + ), + #{succeed => Succ, in_progress => InPro, failed => Failed}. + +is_succeed_result(Result) -> + case Result of + ok -> + true; + {error, {already_in_cluster, _Node}} -> + true; + _ -> + false + end. diff --git a/apps/emqx_management/src/emqx_mgmt_sup.erl b/apps/emqx_management/src/emqx_mgmt_sup.erl index 713ff87dc..5bd8632c3 100644 --- a/apps/emqx_management/src/emqx_mgmt_sup.erl +++ b/apps/emqx_management/src/emqx_mgmt_sup.erl @@ -33,7 +33,8 @@ init([]) -> _ -> [] end, - {ok, {{one_for_one, 1, 5}, Workers}}. + Cluster = child_spec(emqx_mgmt_cluster, 5000, worker), + {ok, {{one_for_one, 1, 5}, [Cluster | Workers]}}. child_spec(Mod, Shutdown, Type) -> #{ diff --git a/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl index 8110ac2cb..b00d63e1d 100644 --- a/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl +++ b/apps/emqx_management/src/proto/emqx_mgmt_cluster_proto_v3.erl @@ -30,7 +30,7 @@ introduced_in() -> "5.5.0". -spec invite_node(node(), node(), timeout()) -> ok | ignore | {error, term()} | emqx_rpc:badrpc(). -invite_node(Node, Self, Timeout) when is_integer(Timeout) -> +invite_node(Node, Self, Timeout) when is_integer(Timeout); Timeout =:= infinity -> rpc:call(Node, emqx_mgmt_api_cluster, join, [Self], Timeout). -spec connected_replicants([node()]) -> emqx_rpc:multicall_result(). diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index 3d4124b28..d195b04e1 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -38,6 +38,9 @@ init_per_testcase(TC = t_cluster_topology_api_replicants, Config0) -> init_per_testcase(TC = t_cluster_invite_api_timeout, Config0) -> Config = [{tc_name, TC} | Config0], [{cluster, cluster(Config)} | setup(Config)]; +init_per_testcase(TC = t_cluster_invite_async, Config0) -> + Config = [{tc_name, TC} | Config0], + [{cluster, cluster(Config)} | setup(Config)]; init_per_testcase(_TC, Config) -> emqx_mgmt_api_test_util:init_suite(?APPS), Config. @@ -48,6 +51,9 @@ end_per_testcase(t_cluster_topology_api_replicants, Config) -> end_per_testcase(t_cluster_invite_api_timeout, Config) -> emqx_cth_cluster:stop(?config(cluster, Config)), cleanup(Config); +end_per_testcase(t_cluster_invite_async, Config) -> + emqx_cth_cluster:stop(?config(cluster, Config)), + cleanup(Config); end_per_testcase(_TC, _Config) -> emqx_mgmt_api_test_util:end_suite(?APPS). @@ -164,6 +170,98 @@ t_cluster_invite_api_timeout(Config) -> lists:sort(Core1Resp3) ). +t_cluster_invite_async(Config) -> + %% assert the cluster is created + [Core1, Core2, Replicant] = _NodesList = ?config(cluster, Config), + {200, Core1Resp} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := + [#{node := Replicant, streams := _}] + }, + #{ + core_node := Core2, + replicant_nodes := + [#{node := Replicant, streams := _}] + } + ], + lists:sort(Core1Resp) + ), + + %% force leave the core2 and replicant + {204} = rpc:call( + Core1, + emqx_mgmt_api_cluster, + force_leave, + [delete, #{bindings => #{node => atom_to_binary(Core2)}}] + ), + %% assert the cluster is updated + {200, Core1Resp2} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := [_] + } + ], + lists:sort(Core1Resp2) + ), + + Invite = fun(Node) -> + Node1 = atom_to_binary(Node), + rpc:call( + Core1, + emqx_mgmt_api_cluster, + invite_node_async, + [put, #{bindings => #{node => Node1}}] + ) + end, + + %% parameter checking + ?assertMatch( + {400, #{code := 'BAD_REQUEST', message := <<"Can't invite self">>}}, + Invite(Core1) + ), + ?assertMatch( + {200}, + Invite(Core2) + ), + %% already invited + ?assertMatch( + {400, #{ + code := 'BAD_REQUEST', + message := <<"The invitation task already created for this node">> + }}, + Invite(Core2) + ), + + %% assert: core2 is in_progress status + ?assertMatch( + {200, #{in_progress := [#{node := Core2}]}}, + rpc:call(Core1, emqx_mgmt_api_cluster, get_invitation_view, [get, #{}]) + ), + + %% waiting the async invitation_succeed + ?assertMatch({succeed, _}, waiting_the_async_invitation_succeed(Core1, Core2)), + + {200, Core1Resp3} = rpc:call(Core1, emqx_mgmt_api_cluster, cluster_topology, [get, #{}]), + ?assertMatch( + [ + #{ + core_node := Core1, + replicant_nodes := + [#{node := Replicant, streams := _}] + }, + #{ + core_node := Core2, + replicant_nodes := _ + } + ], + lists:sort(Core1Resp3) + ). + cluster(Config) -> NodeSpec = #{apps => ?APPS}, Nodes = emqx_cth_cluster:start( @@ -186,3 +284,37 @@ cleanup(Config) -> work_dir(Config) -> filename:join(?config(priv_dir, Config), ?config(tc_name, Config)). + +waiting_the_async_invitation_succeed(Node, TargetNode) -> + waiting_the_async_invitation_succeed(Node, TargetNode, 100). + +waiting_the_async_invitation_succeed(_Node, _TargetNode, 0) -> + error(timeout); +waiting_the_async_invitation_succeed(Node, TargetNode, N) -> + {200, #{ + in_progress := InProgress, + succeed := Succeed, + failed := Failed + }} = rpc:call(Node, emqx_mgmt_api_cluster, get_invitation_view, [get, #{}]), + case find_node_info_list(TargetNode, InProgress) of + error -> + case find_node_info_list(TargetNode, Succeed) of + error -> + case find_node_info_list(TargetNode, Failed) of + error -> error; + Info1 -> {failed, Info1} + end; + Info2 -> + {succeed, Info2} + end; + _Info -> + timer:sleep(1000), + waiting_the_async_invitation_succeed(Node, TargetNode, N - 1) + end. + +find_node_info_list(Node, List) -> + L = lists:filter(fun(#{node := N}) -> N =:= Node end, List), + case L of + [] -> error; + [Info] -> Info + end. diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon index 0e6fceba1..9b7d782fe 100644 --- a/rel/i18n/emqx_mgmt_api_cluster.hocon +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -15,6 +15,11 @@ invite_node.desc: invite_node.label: """Invite node to cluster""" +invite_node_async.desc: +"""Asynchronously invite the target node to join the cluster""" +invite_node_async.label: +"""Asynchronously invite the target node to join the cluster""" + force_remove_node.desc: """Force leave node from cluster""" force_remove_node.label: From 944137ad459b647d7f92b2c00fbe3572f7c60c52 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Jan 2024 16:45:35 +0800 Subject: [PATCH 07/62] chore: format error message --- apps/emqx_management/src/emqx_mgmt_cluster.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index b5dfaae93..9538ba17e 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -169,7 +169,7 @@ state_to_invitation_view(State) -> true -> {[Ret | SuccAcc], FailedAcc}; false -> - {SuccAcc, [Ret#{reason => Result} | FailedAcc]} + {SuccAcc, [Ret#{reason => format_error_reason(Result)} | FailedAcc]} end end, {[], []}, @@ -194,3 +194,6 @@ is_succeed_result(Result) -> _ -> false end. + +format_error_reason(Term) -> + iolist_to_binary(io_lib:format("~p", [Msg])). From 81209e0ded279ce8f93042d39f585d0959d2b41a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Jan 2024 16:47:58 +0800 Subject: [PATCH 08/62] chore: update changes --- apps/emqx_management/src/emqx_mgmt_cluster.erl | 2 +- changes/ce/feat-12267.en.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index 9538ba17e..423442569 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -196,4 +196,4 @@ is_succeed_result(Result) -> end. format_error_reason(Term) -> - iolist_to_binary(io_lib:format("~p", [Msg])). + iolist_to_binary(io_lib:format("~p", [Term])). diff --git a/changes/ce/feat-12267.en.md b/changes/ce/feat-12267.en.md index 70d3aec22..fbc88847c 100644 --- a/changes/ce/feat-12267.en.md +++ b/changes/ce/feat-12267.en.md @@ -1,2 +1,4 @@ Add a new `timeout` parameter to the `cluster/:node/invite` interface. Previously the default timeout was 5s which would often be caused by HTTP API calls due to emqx taking too long to join cluster. + +Add a new endpoint `/cluster/:node/invite_async` to support an asynchronous way to invite nodes to join the cluster. From d2991dae03a368dcf801a71362290c38d9a31209 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 12 Jan 2024 17:17:19 +0800 Subject: [PATCH 09/62] chore: make dialyzer happy --- apps/emqx_management/src/emqx_mgmt_cluster.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index 423442569..12c0c8dbc 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -40,7 +40,7 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec invite_async(atom()) -> ok | ignore | {badrpc, any()}. +-spec invite_async(atom()) -> ok | ignore | {error, {already_started, pid()}}. invite_async(Node) -> JoinTo = node(), case Node =/= JoinTo of From 93ef6766ef10f25f10134c2ae1a1edcc2a756634 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 13 Jan 2024 14:26:31 +0800 Subject: [PATCH 10/62] chore: apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- .../src/emqx_mgmt_api_cluster.erl | 16 ++++++++-------- apps/emqx_management/src/emqx_mgmt_cluster.erl | 6 +++--- changes/ce/feat-12267.en.md | 3 ++- rel/i18n/emqx_mgmt_api_cluster.hocon | 4 ++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index 1a46c0b36..d14b372ab 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -27,7 +27,7 @@ cluster_topology/2, invite_node/2, invite_node_async/2, - get_invitation_view/2, + get_invitation_status/2, force_leave/2, join/1, connected_replicants/0 @@ -179,7 +179,7 @@ fields(invitation_view) -> {succeed, ?HOCON( ?ARRAY(?REF(node_invitation_succeed)), - #{desc => <<"A list of information about nodes that were successfully invited">>} + #{desc => <<"A list of information about nodes which are successfully invited">>} )}, {in_progress, ?HOCON( @@ -198,7 +198,7 @@ fields(node_invitation_failed) -> {reason, ?HOCON( binary(), - #{desc => <<"Failed reason">>, example => <<"Bad RPC to target node">>} + #{desc => <<"Failure reason">>, example => <<"Bad RPC to target node">>} )} ]; fields(node_invitation_succeed) -> @@ -208,7 +208,7 @@ fields(node_invitation_succeed) -> ?HOCON( emqx_utils_calendar:epoch_millisecond(), #{ - desc => <<"The end time of the invitation task, in millisecond">>, + desc => <<"The time of the async invitation result is received, millisecond precision epoch">>, example => <<"1705044829915">> } )} @@ -224,7 +224,7 @@ fields(node_invitation_in_progress) -> ?HOCON( emqx_utils_calendar:epoch_millisecond(), #{ - desc => <<"The start time of the invitation task, in millisecond">>, + desc => <<"The start timestamp of the invitation, millisecond precision epoch">>, example => <<"1705044829915">> } )} @@ -289,15 +289,15 @@ invite_node(put, #{bindings := #{node := Node0}, body := Body}) -> Node = ekka_node:parse_name(binary_to_list(Node0)), case maps:get(<<"timeout">>, Body, ?DEFAULT_INVITE_TIMEOUT) of T when not is_integer(T) -> - {400, #{code => 'BAD_REQUEST', message => <<"timeout must be integer">>}}; + {400, #{code => 'BAD_REQUEST', message => <<"timeout must be an integer">>}}; T when T < 5000 -> - {400, #{code => 'BAD_REQUEST', message => <<"timeout can't less than 5000ms">>}}; + {400, #{code => 'BAD_REQUEST', message => <<"timeout cannot be less than 5000ms">>}}; Timeout -> case emqx_mgmt_cluster_proto_v3:invite_node(Node, node(), Timeout) of ok -> {200}; ignore -> - {400, #{code => 'BAD_REQUEST', message => <<"Can't invite self">>}}; + {400, #{code => 'BAD_REQUEST', message => <<"Cannot invite self">>}}; {badrpc, Error} -> {400, #{code => 'BAD_REQUEST', message => error_message(Error)}}; {error, Error} -> diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index 12c0c8dbc..818e79452 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -45,14 +45,14 @@ invite_async(Node) -> JoinTo = node(), case Node =/= JoinTo of true -> - gen_server:call(?MODULE, {invite_async, Node, JoinTo}); + gen_server:call(?MODULE, {invite_async, Node, JoinTo}, infinity); false -> ignore end. -spec invitation_view() -> map(). invitation_view() -> - gen_server:call(?MODULE, invitation_view). + gen_server:call(?MODULE, invitation_view, infinity). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -196,4 +196,4 @@ is_succeed_result(Result) -> end. format_error_reason(Term) -> - iolist_to_binary(io_lib:format("~p", [Term])). + iolist_to_binary(io_lib:format("~0p", [Term])). diff --git a/changes/ce/feat-12267.en.md b/changes/ce/feat-12267.en.md index fbc88847c..5574918a4 100644 --- a/changes/ce/feat-12267.en.md +++ b/changes/ce/feat-12267.en.md @@ -1,4 +1,5 @@ Add a new `timeout` parameter to the `cluster/:node/invite` interface. Previously the default timeout was 5s which would often be caused by HTTP API calls due to emqx taking too long to join cluster. -Add a new endpoint `/cluster/:node/invite_async` to support an asynchronous way to invite nodes to join the cluster. +Add a new endpoint `/cluster/:node/invite_async` to support an asynchronous way to invite nodes to join the cluster, +and a new endpoint `cluster/invitation` to inspect the join status. diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon index 9b7d782fe..55ce17a9e 100644 --- a/rel/i18n/emqx_mgmt_api_cluster.hocon +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -16,9 +16,9 @@ invite_node.label: """Invite node to cluster""" invite_node_async.desc: -"""Asynchronously invite the target node to join the cluster""" +"""Send a join invitation to a node to join the cluster but do not wait for the join result. Join status can be retrieved with `GET api//invitation`""" invite_node_async.label: -"""Asynchronously invite the target node to join the cluster""" +"""Asynchronously invite""" force_remove_node.desc: """Force leave node from cluster""" From 1668e9ac7d734e150c1709f288df4726c47c53e9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 11 Jan 2024 18:22:50 +0800 Subject: [PATCH 11/62] fix: cannot write literal numbers to influxdb --- apps/emqx_bridge_influxdb/rebar.config | 2 +- .../src/emqx_bridge_influxdb_connector.erl | 119 ++++++++++++++---- .../test/emqx_bridge_influxdb_SUITE.erl | 1 + changes/ee/feat-12247.en.md | 1 + changes/ee/fix-12301.en.md | 1 + mix.exs | 2 +- 6 files changed, 99 insertions(+), 27 deletions(-) create mode 100644 changes/ee/feat-12247.en.md create mode 100644 changes/ee/fix-12301.en.md diff --git a/apps/emqx_bridge_influxdb/rebar.config b/apps/emqx_bridge_influxdb/rebar.config index c6ad26ac1..80d3f8dec 100644 --- a/apps/emqx_bridge_influxdb/rebar.config +++ b/apps/emqx_bridge_influxdb/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ - {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.12"}}}, + {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.13"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 7a84bc440..04fbe01c9 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -59,6 +59,11 @@ -define(DEFAULT_TIMESTAMP_TMPL, "${timestamp}"). +-define(IS_HTTP_ERROR(STATUS_CODE), + (is_integer(STATUS_CODE) andalso + (STATUS_CODE < 200 orelse STATUS_CODE >= 300)) +). + %% ------------------------------------------------------------------------------------------------- %% resource callback callback_mode() -> async_if_possible. @@ -541,7 +546,12 @@ reply_callback(ReplyFunAndArgs, {ok, 401, _, _}) -> ?tp(influxdb_connector_do_query_failure, #{error => <<"authorization failure">>}), Result = {error, {unrecoverable_error, <<"authorization failure">>}}, emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result); +reply_callback(ReplyFunAndArgs, {ok, Code, _, Body}) when ?IS_HTTP_ERROR(Code) -> + ?tp(influxdb_connector_do_query_failure, #{error => Body}), + Result = {error, {unrecoverable_error, Body}}, + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result); reply_callback(ReplyFunAndArgs, Result) -> + ?tp(influxdb_connector_do_query_ok, #{result => Result}), emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). %% ------------------------------------------------------------------------------------------------- @@ -711,56 +721,111 @@ time_unit(ns) -> nanosecond. maps_config_to_data(K, V, {Data, Res}) -> KTransOptions = #{return => rawlist, var_trans => fun key_filter/1}, VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, - NK0 = emqx_placeholder:proc_tmpl(K, Data, KTransOptions), + NK = emqx_placeholder:proc_tmpl(K, Data, KTransOptions), NV = emqx_placeholder:proc_tmpl(V, Data, VTransOptions), - case {NK0, NV} of + case {NK, NV} of {[undefined], _} -> {Data, Res}; %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>] {_, [undefined | _]} -> {Data, Res}; _ -> - NK = list_to_binary(NK0), - {Data, Res#{NK => value_type(NV)}} + {Data, Res#{ + list_to_binary(NK) => value_type(NV, tmpl_type(V)) + }} end. -value_type([Int, <<"i">>]) when - is_integer(Int) --> +value_type([Int, <<"i">>], mixed) when is_integer(Int) -> {int, Int}; -value_type([UInt, <<"u">>]) when - is_integer(UInt) --> +value_type([UInt, <<"u">>], mixed) when is_integer(UInt) -> {uint, UInt}; %% write `1`, `1.0`, `-1.0` all as float %% see also: https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float -value_type([Number]) when is_number(Number) -> - Number; -value_type([<<"t">>]) -> +value_type([Number], _) when is_number(Number) -> + {float, Number}; +value_type([<<"t">>], _) -> 't'; -value_type([<<"T">>]) -> +value_type([<<"T">>], _) -> 'T'; -value_type([true]) -> +value_type([true], _) -> 'true'; -value_type([<<"TRUE">>]) -> +value_type([<<"TRUE">>], _) -> 'TRUE'; -value_type([<<"True">>]) -> +value_type([<<"True">>], _) -> 'True'; -value_type([<<"f">>]) -> +value_type([<<"f">>], _) -> 'f'; -value_type([<<"F">>]) -> +value_type([<<"F">>], _) -> 'F'; -value_type([false]) -> +value_type([false], _) -> 'false'; -value_type([<<"FALSE">>]) -> +value_type([<<"FALSE">>], _) -> 'FALSE'; -value_type([<<"False">>]) -> +value_type([<<"False">>], _) -> 'False'; -value_type(Val) -> - Val. +value_type([Str], variable) when is_binary(Str) -> + Str; +value_type([Str], literal) when is_binary(Str) -> + %% if Str is a literal string suffixed with `i` or `u`, we should convert it to int/uint. + %% otherwise, we should convert it to float. + NumStr = binary:part(Str, 0, byte_size(Str) - 1), + case binary:part(Str, byte_size(Str), -1) of + <<"i">> -> + maybe_convert_to_integer(NumStr, Str, int); + <<"u">> -> + maybe_convert_to_integer(NumStr, Str, uint); + _ -> + maybe_convert_to_float_str(Str) + end; +value_type(Str, _) -> + list_to_binary(Str). + +tmpl_type([{str, _}]) -> + literal; +tmpl_type([{var, _}]) -> + variable; +tmpl_type(_) -> + mixed. + +maybe_convert_to_integer(NumStr, String, Type) -> + try + Int = binary_to_integer(NumStr), + {Type, Int} + catch + error:badarg -> + maybe_convert_to_integer_f(NumStr, String, Type) + end. + +maybe_convert_to_integer_f(NumStr, String, Type) -> + try + Float = binary_to_float(NumStr), + {Type, erlang:floor(Float)} + catch + error:badarg -> + String + end. + +maybe_convert_to_float_str(NumStr) -> + try + _ = binary_to_float(NumStr), + %% NOTE: return a {float, String} to avoid precision loss when converting to float + {float, NumStr} + catch + error:badarg -> + maybe_convert_to_float_str_i(NumStr) + end. + +maybe_convert_to_float_str_i(NumStr) -> + try + _ = binary_to_integer(NumStr), + {float, NumStr} + catch + error:badarg -> + NumStr + end. key_filter(undefined) -> undefined; -key_filter(Value) -> emqx_utils_conv:bin(Value). +key_filter(Value) -> bin(Value). data_filter(undefined) -> undefined; data_filter(Int) when is_integer(Int) -> Int; @@ -799,6 +864,10 @@ str(S) when is_list(S) -> is_unrecoverable_error({error, {unrecoverable_error, _}}) -> true; +is_unrecoverable_error({error, {Code, _}}) when ?IS_HTTP_ERROR(Code) -> + true; +is_unrecoverable_error({error, {Code, _, _Body}}) when ?IS_HTTP_ERROR(Code) -> + true; is_unrecoverable_error(_) -> false. diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl index d79139f17..edb88c72a 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl @@ -945,6 +945,7 @@ t_create_disconnected(Config) -> econnrefused -> ok; closed -> ok; {closed, _} -> ok; + {shutdown, closed} -> ok; _ -> ct:fail("influxdb_client_not_alive with wrong reason: ~p", [Reason]) end, ok diff --git a/changes/ee/feat-12247.en.md b/changes/ee/feat-12247.en.md new file mode 100644 index 000000000..783e8e382 --- /dev/null +++ b/changes/ee/feat-12247.en.md @@ -0,0 +1 @@ +The bridges for InfluxDB have been split so they are available via the connectors and actions APIs. They are still backwards compatible with the old bridge API. diff --git a/changes/ee/fix-12301.en.md b/changes/ee/fix-12301.en.md new file mode 100644 index 000000000..dde764015 --- /dev/null +++ b/changes/ee/fix-12301.en.md @@ -0,0 +1 @@ +Fixed issue where using line protocol to write numeric literals into InfluxDB, but the stored values end up being of string type. diff --git a/mix.exs b/mix.exs index de8195cfb..13d95ccb5 100644 --- a/mix.exs +++ b/mix.exs @@ -199,7 +199,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.5+v0.16.1"}, - {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.12", override: true}, + {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.9.1"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, From e3fee93d9f43e39dd7e1e81b23f60f906a07906c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 12 Jan 2024 20:04:18 +0800 Subject: [PATCH 12/62] fix: write quoted strings to influxdb failed --- .../src/emqx_bridge_influxdb.erl | 2 +- .../src/emqx_bridge_influxdb_connector.erl | 21 ++++- .../test/emqx_bridge_influxdb_SUITE.erl | 56 +++++++++++--- .../test/emqx_bridge_influxdb_tests.erl | 76 +++++++++++++++---- 4 files changed, 124 insertions(+), 31 deletions(-) diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl index 4228d23d5..4fb7dbae7 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl @@ -382,7 +382,7 @@ field(Line) -> field_val([$" | Line]) -> {Val, [$" | Line1]} = unescape(?FIELD_VAL_ESC_CHARS, [$"], Line, []), %% Quoted val can be empty - {Val, strip_l(Line1, ?VAL_SEP)}; + {{quoted, Val}, strip_l(Line1, ?VAL_SEP)}; field_val(Line) -> %% Unquoted value should not be un-escaped according to InfluxDB protocol, %% as it can only hold float, integer, uinteger or boolean value. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 04fbe01c9..478486e5b 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -599,8 +599,17 @@ to_kv_config(KVfields) -> to_maps_config(K, V, Res) -> NK = emqx_placeholder:preproc_tmpl(bin(K)), - NV = emqx_placeholder:preproc_tmpl(bin(V)), - Res#{NK => NV}. + Res#{NK => preproc_quoted(V)}. + +preproc_quoted({quoted, V}) -> + {quoted, emqx_placeholder:preproc_tmpl(bin(V))}; +preproc_quoted(V) -> + emqx_placeholder:preproc_tmpl(bin(V)). + +proc_quoted({quoted, V}, Data, TransOpts) -> + {quoted, emqx_placeholder:proc_tmpl(V, Data, TransOpts)}; +proc_quoted(V, Data, TransOpts) -> + emqx_placeholder:proc_tmpl(V, Data, TransOpts). %% ------------------------------------------------------------------------------------------------- %% Tags & Fields Data Trans @@ -722,19 +731,23 @@ maps_config_to_data(K, V, {Data, Res}) -> KTransOptions = #{return => rawlist, var_trans => fun key_filter/1}, VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, NK = emqx_placeholder:proc_tmpl(K, Data, KTransOptions), - NV = emqx_placeholder:proc_tmpl(V, Data, VTransOptions), + NV = proc_quoted(V, Data, VTransOptions), case {NK, NV} of {[undefined], _} -> {Data, Res}; %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>] {_, [undefined | _]} -> {Data, Res}; + {_, {quoted, [undefined | _]}} -> + {Data, Res}; _ -> {Data, Res#{ list_to_binary(NK) => value_type(NV, tmpl_type(V)) }} end. +value_type({quoted, ValList}, _) -> + {string_list, ValList}; value_type([Int, <<"i">>], mixed) when is_integer(Int) -> {int, Int}; value_type([UInt, <<"u">>], mixed) when is_integer(UInt) -> @@ -778,7 +791,7 @@ value_type([Str], literal) when is_binary(Str) -> maybe_convert_to_float_str(Str) end; value_type(Str, _) -> - list_to_binary(Str). + Str. tmpl_type([{str, _}]) -> literal; diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl index edb88c72a..ee806d826 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl @@ -445,6 +445,7 @@ query_by_clientid(ClientId, Config) -> query => Query, dialect => #{ header => true, + annotations => [<<"datatype">>], delimiter => <<";">> } }), @@ -456,6 +457,7 @@ query_by_clientid(ClientId, Config) -> _Timeout = 10_000, _Retry = 0 ), + %ct:pal("raw body: ~p", [RawBody0]), RawBody1 = iolist_to_binary(string:replace(RawBody0, <<"\r\n">>, <<"\n">>, all)), {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}), DecodedCSV1 = [ @@ -465,21 +467,26 @@ query_by_clientid(ClientId, Config) -> DecodedCSV2 = csv_lines_to_maps(DecodedCSV1), index_by_field(DecodedCSV2). -csv_lines_to_maps([Title | Rest]) -> - csv_lines_to_maps(Rest, Title, _Acc = []); +csv_lines_to_maps([[<<"#datatype">> | DataType], Title | Rest]) -> + csv_lines_to_maps(Rest, Title, _Acc = [], DataType); csv_lines_to_maps([]) -> []. -csv_lines_to_maps([[<<"_result">> | _] = Data | RestData], Title, Acc) -> +csv_lines_to_maps([[<<"_result">> | _] = Data | RestData], Title, Acc, DataType) -> + %ct:pal("data: ~p, title: ~p, datatype: ~p", [Data, Title, DataType]), Map = maps:from_list(lists:zip(Title, Data)), - csv_lines_to_maps(RestData, Title, [Map | Acc]); + MapT = lists:zip(Title, DataType), + [Type] = [T || {<<"_value">>, T} <- MapT], + csv_lines_to_maps(RestData, Title, [Map#{'_value_type' => Type} | Acc], DataType); %% ignore the csv title line %% it's always like this: %% [<<"result">>,<<"table">>,<<"_start">>,<<"_stop">>, %% <<"_time">>,<<"_value">>,<<"_field">>,<<"_measurement">>, Measurement], -csv_lines_to_maps([[<<"result">> | _] = _Title | RestData], Title, Acc) -> - csv_lines_to_maps(RestData, Title, Acc); -csv_lines_to_maps([], _Title, Acc) -> +csv_lines_to_maps([[<<"result">> | _] = _Title | RestData], Title, Acc, DataType) -> + csv_lines_to_maps(RestData, Title, Acc, DataType); +csv_lines_to_maps([[<<"#datatype">> | DataType] | RestData], Title, Acc, _) -> + csv_lines_to_maps(RestData, Title, Acc, DataType); +csv_lines_to_maps([], _Title, Acc, _DataType) -> lists:reverse(Acc). index_by_field(DecodedCSV) -> @@ -494,11 +501,21 @@ assert_persisted_data(ClientId, Expected, PersistedData) -> #{<<"_value">> := ExpectedValue}, maps:get(ClientIdIntKey, PersistedData) ); + (Key, {ExpectedValue, ExpectedType}) -> + ?assertMatch( + #{<<"_value">> := ExpectedValue, '_value_type' := ExpectedType}, + maps:get(atom_to_binary(Key), PersistedData), + #{ + key => Key, + expected_value => ExpectedValue, + expected_data_type => ExpectedType + } + ); (Key, ExpectedValue) -> ?assertMatch( #{<<"_value">> := ExpectedValue}, maps:get(atom_to_binary(Key), PersistedData), - #{expected => ExpectedValue} + #{key => Key, expected_value => ExpectedValue} ) end, Expected @@ -689,7 +706,15 @@ t_const_timestamp(Config) -> Config, #{ <<"write_syntax">> => - <<"mqtt,clientid=${clientid} foo=${payload.foo}i,bar=5i ", ConstBin/binary>> + << + "mqtt,clientid=${clientid} " + "foo=${payload.foo}i," + "foo1=${payload.foo}," + "foo2=\"${payload.foo}\"," + "foo3=\"${payload.foo}somestr\"," + "bar=5i,baz0=1.1,baz1=\"a\",baz2=\"ai\",baz3=\"au\",baz4=\"1u\" ", + ConstBin/binary + >> } ) ), @@ -709,7 +734,18 @@ t_const_timestamp(Config) -> end, ct:sleep(1500), PersistedData = query_by_clientid(ClientId, Config), - Expected = #{foo => <<"123">>}, + Expected = #{ + foo => {<<"123">>, <<"long">>}, + foo1 => {<<"123">>, <<"double">>}, + foo2 => {<<"123">>, <<"string">>}, + foo3 => {<<"123somestr">>, <<"string">>}, + bar => {<<"5">>, <<"long">>}, + baz0 => {<<"1.1">>, <<"double">>}, + baz1 => {<<"a">>, <<"string">>}, + baz2 => {<<"ai">>, <<"string">>}, + baz3 => {<<"au">>, <<"string">>}, + baz4 => {<<"1u">>, <<"string">>} + }, assert_persisted_data(ClientId, Expected, PersistedData), TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"foo">>, PersistedData)), TimeReturned = pad_zero(TimeReturned0), diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl index 9ad685f77..6ddf42f2f 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_tests.erl @@ -102,27 +102,51 @@ #{ measurement => "m7", tags => [{"tag", "tag7"}, {"tag_a", "\"tag7a\""}, {"tag_b", "tag7b"}], - fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b"}], + fields => [ + {"field", {quoted, "field7"}}, + {"field_a", "field7a"}, + {"field_b", {quoted, "field7b"}} + ], timestamp => undefined }}, {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"field8b\" ${timestamp8}", #{ measurement => "m8", tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], - fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "field8b"}], + fields => [ + {"field", {quoted, "field8"}}, + {"field_a", "field8a"}, + {"field_b", {quoted, "field8b"}} + ], timestamp => "${timestamp8}" }}, + { + "m8a,tag=tag8,tag_a=\"${tag8a}\",tag_b=tag8b field=\"${field8}\"," + "field_a=field8a,field_b=\"${field8b}\" ${timestamp8}", + #{ + measurement => "m8a", + tags => [{"tag", "tag8"}, {"tag_a", "\"${tag8a}\""}, {"tag_b", "tag8b"}], + fields => [ + {"field", {quoted, "${field8}"}}, + {"field_a", "field8a"}, + {"field_b", {quoted, "${field8b}"}} + ], + timestamp => "${timestamp8}" + } + }, {"m9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", #{ measurement => "m9", tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], - fields => [{"field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + fields => [ + {"field", {quoted, "field9"}}, {"field_a", "field9a"}, {"field_b", {quoted, ""}} + ], timestamp => "${timestamp9}" }}, {"m10 field=\"\" ${timestamp10}", #{ measurement => "m10", tags => [], - fields => [{"field", ""}], + fields => [{"field", {quoted, ""}}], timestamp => "${timestamp10}" }} ]). @@ -177,19 +201,19 @@ {"m2,tag=tag2 field=\"field \\\"2\\\",\n\"", #{ measurement => "m2", tags => [{"tag", "tag2"}], - fields => [{"field", "field \"2\",\n"}], + fields => [{"field", {quoted, "field \"2\",\n"}}], timestamp => undefined }}, {"m\\ 3 field=\"field3\" ${payload.timestamp\\ 3}", #{ measurement => "m 3", tags => [], - fields => [{"field", "field3"}], + fields => [{"field", {quoted, "field3"}}], timestamp => "${payload.timestamp 3}" }}, {"m4 field=\"\\\"field\\\\4\\\"\"", #{ measurement => "m4", tags => [], - fields => [{"field", "\"field\\4\""}], + fields => [{"field", {quoted, "\"field\\4\""}}], timestamp => undefined }}, { @@ -208,7 +232,11 @@ #{ measurement => "m6", tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + fields => [ + {"field", {quoted, "field6"}}, + {"field_a", {quoted, "field6a"}}, + {"field_b", {quoted, "field6b"}} + ], timestamp => undefined }}, { @@ -217,7 +245,11 @@ #{ measurement => " m7 ", tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}], - fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}], + fields => [ + {"field", {quoted, "field7"}}, + {"field_a", "field7a"}, + {"field_b", {quoted, "field7b\\\n"}} + ], timestamp => undefined } }, @@ -227,7 +259,11 @@ #{ measurement => "m8", tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], - fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}], + fields => [ + {"field", {quoted, "field8"}}, + {"field_a", "field8a"}, + {"field_b", {quoted, "\"field\" = 8b"}} + ], timestamp => "${timestamp8}" } }, @@ -235,14 +271,18 @@ #{ measurement => "m\\9", tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], - fields => [{"field=field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + fields => [ + {"field=field", {quoted, "field9"}}, + {"field_a", "field9a"}, + {"field_b", {quoted, ""}} + ], timestamp => "${timestamp9}" }}, {"m\\,10 \"field\\\\\"=\"\" ${timestamp10}", #{ measurement => "m,10", tags => [], %% backslash should not be un-escaped in tag key - fields => [{"\"field\\\\\"", ""}], + fields => [{"\"field\\\\\"", {quoted, ""}}], timestamp => "${timestamp10}" }} ]). @@ -257,19 +297,19 @@ {" m2,tag=tag2 field=\"field \\\"2\\\",\n\" ", #{ measurement => "m2", tags => [{"tag", "tag2"}], - fields => [{"field", "field \"2\",\n"}], + fields => [{"field", {quoted, "field \"2\",\n"}}], timestamp => undefined }}, {" m\\ 3 field=\"field3\" ${payload.timestamp\\ 3} ", #{ measurement => "m 3", tags => [], - fields => [{"field", "field3"}], + fields => [{"field", {quoted, "field3"}}], timestamp => "${payload.timestamp 3}" }}, {" m4 field=\"\\\"field\\\\4\\\"\" ", #{ measurement => "m4", tags => [], - fields => [{"field", "\"field\\4\""}], + fields => [{"field", {quoted, "\"field\\4\""}}], timestamp => undefined }}, { @@ -288,7 +328,11 @@ #{ measurement => "m6", tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], - fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + fields => [ + {"field", {quoted, "field6"}}, + {"field_a", {quoted, "field6a"}}, + {"field_b", {quoted, "field6b"}} + ], timestamp => undefined }} ]). From e49d3ca50c521ec500a0ea8c7f00710c2845efb1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Jan 2024 02:15:03 +0800 Subject: [PATCH 13/62] feat: support elasticsearch bridge --- apps/emqx_bridge/src/emqx_action_info.erl | 3 +- apps/emqx_bridge_es/.gitignore | 19 + apps/emqx_bridge_es/BSL.txt | 94 ++++ apps/emqx_bridge_es/README.md | 23 + apps/emqx_bridge_es/docker-ct | 1 + apps/emqx_bridge_es/etc/emqx_bridge_es.conf | 0 .../emqx_bridge_es/include/emqx_bridge_es.hrl | 8 + apps/emqx_bridge_es/rebar.config | 15 + .../emqx_bridge_es/src/emqx_bridge_es.app.src | 23 + apps/emqx_bridge_es/src/emqx_bridge_es.erl | 312 +++++++++++ .../src/emqx_bridge_es_action_info.erl | 22 + .../src/emqx_bridge_es_connector.erl | 498 ++++++++++++++++++ .../src/emqx_bridge_http_connector.erl | 28 +- .../src/emqx_bridge_iotdb_connector.erl | 9 +- .../src/schema/emqx_connector_ee_schema.erl | 18 +- .../src/schema/emqx_connector_schema.erl | 4 +- apps/emqx_machine/priv/reboot_lists.eterm | 1 + apps/emqx_machine/src/emqx_machine.app.src | 2 +- mix.exs | 1 + rel/i18n/emqx_bridge_es.hocon | 129 +++++ rel/i18n/emqx_bridge_es_connector.hocon | 44 ++ 21 files changed, 1234 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_bridge_es/.gitignore create mode 100644 apps/emqx_bridge_es/BSL.txt create mode 100644 apps/emqx_bridge_es/README.md create mode 100644 apps/emqx_bridge_es/docker-ct create mode 100644 apps/emqx_bridge_es/etc/emqx_bridge_es.conf create mode 100644 apps/emqx_bridge_es/include/emqx_bridge_es.hrl create mode 100644 apps/emqx_bridge_es/rebar.config create mode 100644 apps/emqx_bridge_es/src/emqx_bridge_es.app.src create mode 100644 apps/emqx_bridge_es/src/emqx_bridge_es.erl create mode 100644 apps/emqx_bridge_es/src/emqx_bridge_es_action_info.erl create mode 100644 apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl create mode 100644 rel/i18n/emqx_bridge_es.hocon create mode 100644 rel/i18n/emqx_bridge_es_connector.hocon diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index ceba8e202..5ce60fe6c 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -87,7 +87,8 @@ hard_coded_action_info_modules_ee() -> emqx_bridge_syskeeper_action_info, emqx_bridge_timescale_action_info, emqx_bridge_redis_action_info, - emqx_bridge_iotdb_action_info + emqx_bridge_iotdb_action_info, + emqx_bridge_es_action_info ]. -else. hard_coded_action_info_modules_ee() -> diff --git a/apps/emqx_bridge_es/.gitignore b/apps/emqx_bridge_es/.gitignore new file mode 100644 index 000000000..e9bc1c544 --- /dev/null +++ b/apps/emqx_bridge_es/.gitignore @@ -0,0 +1,19 @@ +.rebar3 + _* + .eunit + *.o + *.beam + *.plt + *.swp + *.swo + .erlang.cookie + ebin + log + erl_crash.dump + .rebar + logs + _build + .idea + *.iml + rebar3.crashdump + *~ diff --git a/apps/emqx_bridge_es/BSL.txt b/apps/emqx_bridge_es/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_es/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_es/README.md b/apps/emqx_bridge_es/README.md new file mode 100644 index 000000000..b91af4aba --- /dev/null +++ b/apps/emqx_bridge_es/README.md @@ -0,0 +1,23 @@ +# Apache ElasticSearch Data Integration Bridge + +This application houses the ElasticSearch data integration bridge for EMQX Enterprise + Edition. It provides the means to connect to ElasticSearch and publish messages to it. + +It implements the connection management and interaction without need for a + separate connector app, since it's not used by authentication and authorization + applications. + + + +# Contributing +Please see our [contributing.md](../../CONTRIBUTING.md). + +# License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_es/docker-ct b/apps/emqx_bridge_es/docker-ct new file mode 100644 index 000000000..80f0d394b --- /dev/null +++ b/apps/emqx_bridge_es/docker-ct @@ -0,0 +1 @@ +toxiproxy diff --git a/apps/emqx_bridge_es/etc/emqx_bridge_es.conf b/apps/emqx_bridge_es/etc/emqx_bridge_es.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_es/include/emqx_bridge_es.hrl b/apps/emqx_bridge_es/include/emqx_bridge_es.hrl new file mode 100644 index 000000000..8393cff2b --- /dev/null +++ b/apps/emqx_bridge_es/include/emqx_bridge_es.hrl @@ -0,0 +1,8 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_BRIDGE_ES_HRL). +-define(EMQX_BRIDGE_ES_HRL, true). + +-endif. diff --git a/apps/emqx_bridge_es/rebar.config b/apps/emqx_bridge_es/rebar.config new file mode 100644 index 000000000..2a3526e08 --- /dev/null +++ b/apps/emqx_bridge_es/rebar.config @@ -0,0 +1,15 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}}, + {emqx_bridge_http, {path, "../emqx_bridge_http"}} +]}. +{plugins, [rebar3_path_deps]}. +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src new file mode 100644 index 000000000..9e98cd33e --- /dev/null +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -0,0 +1,23 @@ +%% -*- mode: erlang -*- +{application, emqx_bridge_es, [ + {description, "EMQX Enterprise Elastic Search Bridge"}, + {vsn, "0.1.0"}, + {modules, [ + emqx_bridge_es, + emqx_bridge_es_connector + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_connector + ]}, + {env, []}, + {licenses, ["Business Source License 1.1"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl new file mode 100644 index 000000000..57ab648b5 --- /dev/null +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -0,0 +1,312 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_es). + +-include("emqx_bridge_es.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-export([bridge_v2_examples/1]). + +%% hocon_schema API +-export([namespace/0, roots/0, fields/1, desc/1]). + +-define(CONNECTOR_TYPE, elasticsearch). +-define(ACTION_TYPE, ?CONNECTOR_TYPE). + +namespace() -> "bridge_elasticsearch". + +roots() -> []. + +fields(action) -> + {elasticsearch, + ?HOCON( + ?MAP(action_name, ?R_REF(action_config)), + #{ + desc => <<"ElasticSearch Action Config">>, + required => false + } + )}; +fields(action_config) -> + emqx_resource_schema:override( + emqx_bridge_v2_schema:make_producer_action_schema( + ?HOCON( + ?R_REF(action_parameters), + #{ + required => true, desc => ?DESC("action_parameters") + } + ) + ), + [ + {resource_opts, + ?HOCON(?R_REF(action_resource_opts), #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ] + ); +fields(action_resource_opts) -> + lists:filter( + fun({K, _V}) -> + not lists:member(K, unsupported_opts()) + end, + emqx_bridge_v2_schema:resource_opts_fields() + ); +fields(action_parameters) -> + [ + {target, + ?HOCON( + binary(), + #{ + desc => ?DESC("config_target"), + required => false + } + )}, + {require_alias, + ?HOCON( + boolean(), + #{ + required => false, + default => false, + desc => ?DESC("config_require_alias") + } + )}, + {routing, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_routing") + } + )}, + {wait_for_active_shards, + ?HOCON( + ?UNION([pos_integer(), all]), + #{ + required => false, + desc => ?DESC("config_wait_for_active_shards") + } + )}, + {data, + ?HOCON( + ?ARRAY( + ?UNION( + [ + ?R_REF(create), + ?R_REF(delete), + ?R_REF(index), + ?R_REF(update) + ] + ) + ), + #{ + desc => ?DESC("action_parameters_data") + } + )} + ] ++ + lists:filter( + fun({K, _}) -> + not lists:member(K, [path, method, body, headers, request_timeout]) + end, + emqx_bridge_http_schema:fields("parameters_opts") + ); +fields(Action) when Action =:= create; Action =:= index -> + [ + {action, + ?HOCON( + Action, + #{ + desc => atom_to_binary(Action), + required => true + } + )}, + {'_index', + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_index") + } + )}, + {'_id', + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_id") + } + )}, + {require_alias, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_require_alias") + } + )}, + {fields, + ?HOCON( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_fields") + } + )} + ]; +fields(delete) -> + [ + {action, + ?HOCON( + delete, + #{ + desc => <<"Delete">>, + required => true + } + )}, + {'_index', + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_index") + } + )}, + {'_id', + ?HOCON( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_id") + } + )}, + {require_alias, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_require_alias") + } + )} + ]; +fields(update) -> + [ + {action, + ?HOCON( + update, + #{ + desc => <<"Update">>, + required => true + } + )}, + {doc_as_upsert, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_doc_as_upsert") + } + )}, + {upsert, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_upsert") + } + )}, + {'_index', + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_index") + } + )}, + {'_id', + ?HOCON( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_id") + } + )}, + {require_alias, + ?HOCON( + binary(), + #{ + required => false, + desc => ?DESC("config_parameters_require_alias") + } + )}, + {fields, + ?HOCON( + binary(), + #{ + required => true, + desc => ?DESC("config_parameters_fields") + } + )} + ]; +fields("post_bridge_v2") -> + emqx_bridge_schema:type_and_name_fields(elasticsearch) ++ fields(action_config); +fields("put_bridge_v2") -> + fields(action_config); +fields("get_bridge_v2") -> + emqx_bridge_schema:status_fields() ++ fields("post_bridge_v2"). + +bridge_v2_examples(Method) -> + [ + #{ + <<"elasticsearch">> => + #{ + summary => <<"Elastic Search Bridge">>, + value => emqx_bridge_v2_schema:action_values( + Method, ?ACTION_TYPE, ?CONNECTOR_TYPE, action_values() + ) + } + } + ]. + +action_values() -> + #{ + parameters => #{ + target => <<"${target_index}">>, + data => [ + #{ + action => index, + '_index' => <<"${index}">>, + fields => <<"${fields}">>, + require_alias => <<"${require_alias}">> + }, + #{ + action => create, + '_index' => <<"${index}">>, + fields => <<"${fields}">> + }, + #{ + action => delete, + '_index' => <<"${index}">>, + '_id' => <<"${id}">> + }, + #{ + action => update, + '_index' => <<"${index}">>, + '_id' => <<"${id}">>, + fields => <<"${fields}">>, + require_alias => false, + doc_as_upsert => <<"${doc_as_upsert}">>, + upsert => <<"${upsert}">> + } + ] + } + }. + +unsupported_opts() -> + [ + batch_size, + batch_time + ]. + +desc(_) -> undefined. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_action_info.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_action_info.erl new file mode 100644 index 000000000..b2f2ff777 --- /dev/null +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_action_info.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_es_action_info). + +-behaviour(emqx_action_info). + +-elvis([{elvis_style, invalid_dynamic_call, disable}]). + +%% behaviour callbacks +-export([ + action_type_name/0, + connector_type_name/0, + schema_module/0 +]). + +-define(ACTION_TYPE, elasticsearch). + +action_type_name() -> ?ACTION_TYPE. +connector_type_name() -> ?ACTION_TYPE. + +schema_module() -> emqx_bridge_es. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl new file mode 100644 index 000000000..22509e037 --- /dev/null +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -0,0 +1,498 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_es_connector). + +-behaviour(emqx_resource). + +-include("emqx_bridge_es.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + callback_mode/0, + on_start/2, + on_stop/2, + on_get_status/2, + on_query/3, + on_query_async/4, + on_add_channel/4, + on_remove_channel/3, + on_get_channels/1, + on_get_channel_status/3 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1, + connector_examples/1, + connector_example_values/0 +]). + +%% emqx_connector_resource behaviour callbacks +-export([connector_config/2]). + +-type config() :: + #{ + base_url := #{ + scheme := http | https, + host := iolist(), + port := inet:port_number(), + path := _ + }, + connect_timeout := pos_integer(), + pool_type := random | hash, + pool_size := pos_integer(), + request => undefined | map(), + atom() => _ + }. + +-type state() :: + #{ + base_path := _, + connect_timeout := pos_integer(), + pool_type := random | hash, + channels := map(), + request => undefined | map(), + atom() => _ + }. + +-type manager_id() :: binary(). + +-define(CONNECTOR_TYPE, elasticsearch). + +%%------------------------------------------------------------------------------------- +%% connector examples +%%------------------------------------------------------------------------------------- +connector_examples(Method) -> + [ + #{ + <<"elasticsearch">> => + #{ + summary => <<"Elastic Search Connector">>, + value => emqx_connector_schema:connector_values( + Method, ?CONNECTOR_TYPE, connector_example_values() + ) + } + } + ]. + +connector_example_values() -> + #{ + name => <<"elasticsearch_connector">>, + type => elasticsearch, + enable => true, + authentication => #{ + <<"username">> => <<"root">>, + <<"password">> => <<"******">> + }, + base_url => <<"http://127.0.0.1:9200/">>, + connect_timeout => <<"15s">>, + pool_type => <<"random">>, + pool_size => 8, + enable_pipelining => 100, + ssl => #{enable => false} + }. + +%%------------------------------------------------------------------------------------- +%% schema +%%------------------------------------------------------------------------------------- +namespace() -> "elasticsearch". + +roots() -> + [{config, #{type => ?R_REF(config)}}]. + +fields(config) -> + lists:filter( + fun({K, _}) -> not lists:member(K, [url, request, retry_interval, headers]) end, + emqx_bridge_http_schema:fields("config_connector") + ) ++ + fields("connection_fields"); +fields("connection_fields") -> + [ + {base_url, + ?HOCON( + emqx_schema:url(), + #{ + required => true, + desc => ?DESC(emqx_bridge_es, "config_base_url") + } + )}, + {authentication, + ?HOCON( + ?UNION([?R_REF(auth_basic)]), + #{ + desc => ?DESC("config_authentication") + } + )} + ]; +fields(auth_basic) -> + [ + {username, + ?HOCON(binary(), #{ + required => true, + desc => ?DESC("config_auth_basic_username") + })}, + {password, + emqx_schema_secret:mk(#{ + required => true, + desc => ?DESC("config_auth_basic_password") + })} + ]; +fields("post") -> + emqx_connector_schema:type_and_name_fields(elasticsearch) ++ fields(config); +fields("put") -> + fields(config); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc(config) -> + ?DESC("desc_config"); +desc(auth_basic) -> + "Basic Authentication"; +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Elastic Search using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +connector_config(Conf, #{name := Name, parse_confs := ParseConfs}) -> + #{ + base_url := BaseUrl, + authentication := + #{ + username := Username, + password := Password0 + } + } = Conf, + + Password = emqx_secret:unwrap(Password0), + Base64 = base64:encode(<>), + BasicToken = <<"Basic ", Base64/binary>>, + + WebhookConfig = + Conf#{ + method => <<"post">>, + url => BaseUrl, + headers => [ + {<<"Content-type">>, <<"application/json">>}, + {<<"Authorization">>, BasicToken} + ] + }, + ParseConfs( + <<"http">>, + Name, + WebhookConfig + ). + +%%------------------------------------------------------------------------------------- +%% `emqx_resource' API +%%------------------------------------------------------------------------------------- +callback_mode() -> async_if_possible. + +-spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). +on_start(InstanceId, Config) -> + case emqx_bridge_http_connector:on_start(InstanceId, Config) of + {ok, State} -> + ?SLOG(info, #{ + msg => "elasticsearch_bridge_started", + instance_id => InstanceId, + request => maps:get(request, State, <<>>) + }), + ?tp(elasticsearch_bridge_started, #{instance_id => InstanceId}), + {ok, State#{channels => #{}}}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_start_elasticsearch_bridge", + instance_id => InstanceId, + request => maps:get(request, Config, <<>>), + reason => Reason + }), + throw(failed_to_start_elasticsearch_bridge) + end. + +-spec on_stop(manager_id(), state()) -> ok | {error, term()}. +on_stop(InstanceId, State) -> + ?SLOG(info, #{ + msg => "stopping_elasticsearch_bridge", + connector => InstanceId + }), + Res = emqx_bridge_http_connector:on_stop(InstanceId, State), + ?tp(elasticsearch_bridge_stopped, #{instance_id => InstanceId}), + Res. + +-spec on_get_status(manager_id(), state()) -> + {connected, state()} | {disconnected, state(), term()}. +on_get_status(InstanceId, State) -> + emqx_bridge_http_connector:on_get_status(InstanceId, State). + +-spec on_query(manager_id(), tuple(), state()) -> + {ok, pos_integer(), [term()], term()} + | {ok, pos_integer(), [term()]} + | {error, term()}. +on_query(InstanceId, {ChannelId, Msg} = Req, #{channels := Channels} = State) -> + ?tp(elasticsearch_bridge_on_query, #{instance_id => InstanceId}), + ?SLOG(debug, #{ + msg => "elasticsearch_bridge_on_query_called", + instance_id => InstanceId, + send_message => Req, + state => emqx_utils:redact(State) + }), + case try_render_message(Req, Channels) of + {ok, Body} -> + handle_response( + emqx_bridge_http_connector:on_query( + InstanceId, {ChannelId, {Msg, Body}}, State + ) + ); + Error -> + Error + end. + +-spec on_query_async(manager_id(), tuple(), {function(), [term()]}, state()) -> + {ok, pid()} | {error, empty_request}. +on_query_async( + InstanceId, {ChannelId, Msg} = Req, ReplyFunAndArgs0, #{channels := Channels} = State +) -> + ?tp(elasticsearch_bridge_on_query_async, #{instance_id => InstanceId}), + ?SLOG(debug, #{ + msg => "elasticsearch_bridge_on_query_async_called", + instance_id => InstanceId, + send_message => Req, + state => emqx_utils:redact(State) + }), + case try_render_message(Req, Channels) of + {ok, Payload} -> + ReplyFunAndArgs = + { + fun(Result) -> + Response = handle_response(Result), + emqx_resource:apply_reply_fun(ReplyFunAndArgs0, Response) + end, + [] + }, + emqx_bridge_http_connector:on_query_async( + InstanceId, {ChannelId, {Msg, Payload}}, ReplyFunAndArgs, State + ); + Error -> + Error + end. + +on_add_channel( + InstanceId, + #{channels := Channels} = State0, + ChannelId, + #{parameters := Parameter} +) -> + case maps:is_key(ChannelId, Channels) of + true -> + {error, already_exists}; + _ -> + #{data := Data} = Parameter, + Parameter1 = Parameter#{path => path(Parameter), method => <<"post">>}, + {ok, State} = emqx_bridge_http_connector:on_add_channel( + InstanceId, State0, ChannelId, #{parameters => Parameter1} + ), + case preproc_data_template(Data) of + [] -> + {error, invalid_data}; + DataTemplate -> + Channel = Parameter1#{data => DataTemplate}, + Channels2 = Channels#{ChannelId => Channel}, + {ok, State#{channels => Channels2}} + end + end. + +on_remove_channel(InstanceId, #{channels := Channels} = OldState0, ChannelId) -> + {ok, OldState} = emqx_bridge_http_connector:on_remove_channel(InstanceId, OldState0, ChannelId), + Channels2 = maps:remove(ChannelId, Channels), + {ok, OldState#{channels => Channels2}}. + +on_get_channels(InstanceId) -> + emqx_bridge_v2:get_channels_for_connector(InstanceId). + +on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> + case maps:is_key(ChannelId, Channels) of + true -> + connected; + _ -> + {error, not_exists} + end. + +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- +path(Param) -> + Target = maps:get(target, Param, undefined), + QString0 = maps:fold( + fun(K, V, Acc) -> + [[atom_to_list(K), "=", to_str(V)] | Acc] + end, + [["_source=false"], ["filter_path=items.*.error"]], + maps:with([require_alias, routing, wait_for_active_shards], Param) + ), + QString = "?" ++ lists:join("&", QString0), + target(Target) ++ QString. + +target(undefined) -> "/_bulk"; +target(Str) -> "/" ++ binary_to_list(Str) ++ "/_bulk". + +to_str(List) when is_list(List) -> List; +to_str(false) -> "false"; +to_str(true) -> "true"; +to_str(Atom) when is_atom(Atom) -> atom_to_list(Atom). + +proc_data(DataList, Msg) when is_list(DataList) -> + [ + begin + proc_data(Data, Msg) + end + || Data <- DataList + ]; +proc_data( + #{ + action := Action, + '_index' := IndexT, + '_id' := IdT, + require_alias := RequiredAliasT, + fields := FieldsT + }, + Msg +) when Action =:= create; Action =:= index -> + [ + emqx_utils_json:encode( + #{ + Action => filter([ + {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, + {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, + {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)} + ]) + } + ), + "\n", + emqx_placeholder:proc_tmpl(FieldsT, Msg), + "\n" + ]; +proc_data( + #{ + action := delete, + '_index' := IndexT, + '_id' := IdT, + require_alias := RequiredAliasT + }, + Msg +) -> + [ + emqx_utils_json:encode( + #{ + delete => filter([ + {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, + {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, + {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)} + ]) + } + ), + "\n" + ]; +proc_data( + #{ + action := update, + '_index' := IndexT, + '_id' := IdT, + require_alias := RequiredAliasT, + doc_as_upsert := DocAsUpsert, + upsert := Upsert, + fields := FieldsT + }, + Msg +) -> + [ + emqx_utils_json:encode( + #{ + update => filter([ + {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, + {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, + {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)}, + {doc_as_upsert, emqx_placeholder:proc_tmpl(DocAsUpsert, Msg)}, + {upsert, emqx_placeholder:proc_tmpl(Upsert, Msg)} + ]) + } + ), + "\n{\"doc\":", + emqx_placeholder:proc_tmpl(FieldsT, Msg), + "}\n" + ]. + +filter(List) -> + Fun = fun + ({_K, V}) when V =:= undefined; V =:= <<"undefined">>; V =:= "undefined" -> + false; + ({_K, V}) when V =:= ""; V =:= <<>> -> + false; + ({_K, V}) when V =:= "false" -> {true, false}; + ({_K, V}) when V =:= "true" -> {true, true}; + ({_K, _V}) -> + true + end, + maps:from_list(lists:filtermap(Fun, List)). + +handle_response({ok, 200, _Headers, Body} = Resp) -> + eval_response_body(Body, Resp); +handle_response({ok, 200, Body} = Resp) -> + eval_response_body(Body, Resp); +handle_response({ok, Code, _Headers, Body}) -> + {error, #{code => Code, body => Body}}; +handle_response({ok, Code, Body}) -> + {error, #{code => Code, body => Body}}; +handle_response({error, _} = Error) -> + Error. + +eval_response_body(<<"{}">>, Resp) -> Resp; +eval_response_body(Body, _Resp) -> {error, emqx_utils_json:decode(Body)}. + +preproc_data_template(DataList) when is_list(DataList) -> + [ + begin + preproc_data_template(Data) + end + || Data <- DataList + ]; +preproc_data_template(#{action := create} = Data) -> + Index = maps:get('_index', Data, ""), + Id = maps:get('_id', Data, ""), + RequiredAlias = maps:get(require_alias, Data, ""), + Fields = maps:get(fields, Data, ""), + #{ + action => create, + '_index' => emqx_placeholder:preproc_tmpl(Index), + '_id' => emqx_placeholder:preproc_tmpl(Id), + require_alias => emqx_placeholder:preproc_tmpl(RequiredAlias), + fields => emqx_placeholder:preproc_tmpl(Fields) + }; +preproc_data_template(#{action := index} = Data) -> + Data1 = preproc_data_template(Data#{action => create}), + Data1#{action => index}; +preproc_data_template(#{action := delete} = Data) -> + Data1 = preproc_data_template(Data#{action => create}), + Data2 = Data1#{action => delete}, + maps:remove(fields, Data2); +preproc_data_template(#{action := update} = Data) -> + Data1 = preproc_data_template(Data#{action => index}), + DocAsUpsert = maps:get(doc_as_upsert, Data, ""), + Upsert = maps:get(upsert, Data, ""), + Data1#{ + action => update, + doc_as_upsert => emqx_placeholder:preproc_tmpl(DocAsUpsert), + upsert => emqx_placeholder:preproc_tmpl(Upsert) + }. + +try_render_message({ChannelId, Msg}, Channels) -> + case maps:find(ChannelId, Channels) of + {ok, #{data := Data}} -> + {ok, proc_data(Data, Msg)}; + _ -> + {error, {unrecoverable_error, {invalid_channel_id, ChannelId}}} + end. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 51375fc04..f00ae8523 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -317,7 +317,7 @@ on_query(InstId, {send_message, Msg}, State) -> %% BridgeV2 entrypoint on_query( InstId, - {ActionId, Msg}, + {ActionId, MsgAndBody}, State = #{installed_actions := InstalledActions} ) when is_binary(ActionId) -> case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of @@ -334,10 +334,10 @@ on_query( body := Body, headers := Headers, request_timeout := Timeout - } = process_request_and_action(Request, ActionState, Msg), + } = process_request_and_action(Request, ActionState, MsgAndBody), %% bridge buffer worker has retry, do not let ehttpc retry Retry = 2, - ClientId = maps:get(clientid, Msg, undefined), + ClientId = clientid(MsgAndBody), on_query( InstId, {ClientId, Method, {Path, Headers, Body}, Timeout, Retry}, @@ -430,7 +430,7 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> %% BridgeV2 entrypoint on_query_async( InstId, - {ActionId, Msg}, + {ActionId, MsgAndBody}, ReplyFunAndArgs, State = #{installed_actions := InstalledActions} ) when is_binary(ActionId) -> @@ -448,8 +448,8 @@ on_query_async( body := Body, headers := Headers, request_timeout := Timeout - } = process_request_and_action(Request, ActionState, Msg), - ClientId = maps:get(clientid, Msg, undefined), + } = process_request_and_action(Request, ActionState, MsgAndBody), + ClientId = clientid(MsgAndBody), on_query_async( InstId, {ClientId, Method, {Path, Headers, Body}, Timeout}, @@ -629,12 +629,9 @@ maybe_parse_template(Key, Conf) -> parse_template(String) -> emqx_template:parse(String). -process_request_and_action(Request, ActionState, Msg) -> +process_request_and_action(Request, ActionState, {Msg, Body}) -> MethodTemplate = maps:get(method, ActionState), Method = make_method(render_template_string(MethodTemplate, Msg)), - BodyTemplate = maps:get(body, ActionState), - Body = render_request_body(BodyTemplate, Msg), - PathPrefix = unicode:characters_to_list(render_template(maps:get(path, Request), Msg)), PathSuffix = unicode:characters_to_list(render_template(maps:get(path, ActionState), Msg)), @@ -656,7 +653,11 @@ process_request_and_action(Request, ActionState, Msg) -> body => Body, headers => Headers, request_timeout => maps:get(request_timeout, ActionState) - }. + }; +process_request_and_action(Request, ActionState, Msg) -> + BodyTemplate = maps:get(body, ActionState), + Body = render_request_body(BodyTemplate, Msg), + process_request_and_action(Request, ActionState, {Msg, Body}). merge_proplist(Proplist1, Proplist2) -> lists:foldl( @@ -732,7 +733,7 @@ formalize_request(_Method, BasePath, {Path, Headers}) -> %% because an HTTP server may handle paths like %% "/a/b/c/", "/a/b/c" and "/a//b/c" differently. %% -%% So we try to avoid unneccessary path normalization. +%% So we try to avoid unnecessary path normalization. %% %% See also: `join_paths_test_/0` join_paths(Path1, Path2) -> @@ -876,6 +877,9 @@ redact_request({Path, Headers}) -> redact_request({Path, Headers, _Body}) -> {Path, Headers, <<"******">>}. +clientid({Msg, _Body}) -> clientid(Msg); +clientid(Msg) -> maps:get(clientid, Msg, undefined). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 4286a59e4..7fbcfc6db 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -90,7 +90,7 @@ connector_example_values() -> enable => true, authentication => #{ <<"username">> => <<"root">>, - <<"password">> => <<"*****">> + <<"password">> => <<"******">> }, base_url => <<"http://iotdb.local:18080/">>, connect_timeout => <<"15s">>, @@ -109,7 +109,10 @@ roots() -> [{config, #{type => hoconsc:ref(?MODULE, config)}}]. fields(config) -> - proplists_without([url, headers], emqx_bridge_http_schema:fields("config_connector")) ++ + proplists_without( + [url, request, retry_interval, headers], + emqx_bridge_http_schema:fields("config_connector") + ) ++ fields("connection_fields"); fields("connection_fields") -> [ @@ -206,7 +209,7 @@ on_start(InstanceId, Config) -> ?SLOG(error, #{ msg => "failed_to_start_iotdb_bridge", instance_id => InstanceId, - base_url => maps:get(request, Config, <<>>), + request => maps:get(request, Config, <<>>), reason => Reason }), throw(failed_to_start_iotdb_bridge) diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 9e35015b0..822e8429b 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -50,6 +50,8 @@ resource_type(redis) -> emqx_bridge_redis_connector; resource_type(iotdb) -> emqx_bridge_iotdb_connector; +resource_type(elasticsearch) -> + emqx_bridge_es_connector; resource_type(Type) -> error({unknown_connector_type, Type}). @@ -62,6 +64,8 @@ connector_impl_module(confluent_producer) -> emqx_bridge_confluent_producer; connector_impl_module(iotdb) -> emqx_bridge_iotdb_connector; +connector_impl_module(elasticsearch) -> + emqx_bridge_es_connector; connector_impl_module(_ConnectorType) -> undefined. @@ -181,6 +185,14 @@ connector_structs() -> desc => <<"IoTDB Connector Config">>, required => false } + )}, + {elasticsearch, + mk( + hoconsc:map(name, ref(emqx_bridge_es_connector, config)), + #{ + desc => <<"Elastis Search Connector Config">>, + required => false + } )} ]. @@ -199,7 +211,8 @@ schema_modules() -> emqx_bridge_timescale, emqx_postgresql_connector_schema, emqx_bridge_redis_schema, - emqx_bridge_iotdb_connector + emqx_bridge_iotdb_connector, + emqx_bridge_es_connector ]. api_schemas(Method) -> @@ -227,7 +240,8 @@ api_schemas(Method) -> api_ref(emqx_bridge_timescale, <<"timescale">>, Method ++ "_connector"), api_ref(emqx_postgresql_connector_schema, <<"pgsql">>, Method ++ "_connector"), api_ref(emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector"), - api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method) + api_ref(emqx_bridge_iotdb_connector, <<"iotdb">>, Method), + api_ref(emqx_bridge_es_connector, <<"elasticsearch">>, Method) ]. api_ref(Module, Type, Method) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 0a3c9d744..b043ebacd 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -147,7 +147,9 @@ connector_type_to_bridge_types(syskeeper_proxy) -> connector_type_to_bridge_types(timescale) -> [timescale]; connector_type_to_bridge_types(iotdb) -> - [iotdb]. + [iotdb]; +connector_type_to_bridge_types(elasticsearch) -> + [elasticsearch]. actions_config_name() -> <<"actions">>. diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index cf3ad1523..f7e78c360 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -99,6 +99,7 @@ emqx_bridge_hstreamdb, emqx_bridge_influxdb, emqx_bridge_iotdb, + emqx_bridge_es, emqx_bridge_matrix, emqx_bridge_mongodb, emqx_bridge_mysql, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 6d7012313..320ddea02 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.17"}, + {vsn, "0.2.18"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/mix.exs b/mix.exs index de8195cfb..66889cebf 100644 --- a/mix.exs +++ b/mix.exs @@ -164,6 +164,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, :emqx_bridge_iotdb, + :emqx_bridge_es, :emqx_bridge_matrix, :emqx_bridge_mongodb, :emqx_bridge_mysql, diff --git a/rel/i18n/emqx_bridge_es.hocon b/rel/i18n/emqx_bridge_es.hocon new file mode 100644 index 000000000..78299c4ee --- /dev/null +++ b/rel/i18n/emqx_bridge_es.hocon @@ -0,0 +1,129 @@ +emqx_bridge_es { + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +config_authentication.desc: +"""Authentication configuration""" + +config_authentication.label: +"""Authentication""" + +auth_basic.desc: +"""Parameters for basic authentication.""" + +auth_basic.label: +"""Basic auth params""" + +config_auth_basic_username.desc: +"""The username as configured at the IoTDB REST interface""" + +config_auth_basic_username.label: + """HTTP Basic Auth Username""" + +config_auth_basic_password.desc: +"""The password as configured at the IoTDB REST interface""" + +config_auth_basic_password.label: +"""HTTP Basic Auth Password""" + +config_base_url.desc: +"""The base URL of the external ElasticSearch service's REST interface.""" +config_base_url.label: +"""ElasticSearch REST Service Base URL""" + +config_target.desc: +"""Name of the data stream, index, or index alias to perform bulk actions on""" + +config_target.label: +"""Target""" + +config_require_alias.desc: +"""If true, the request’s actions must target an index alias. Defaults to false""" +config_require_alias.label: +"""Require Alias""" + +config_routing.desc: +"""Custom value used to route operations to a specific shard.""" +config_routing.label: +"""Routing""" + +config_wait_for_active_shards.desc: +"""The number of shard copies that must be active before proceeding with the operation. +Set to all or any positive integer up to the total number of shards in the index (number_of_replicas+1). +Default: 1, the primary shard""" + +config_max_retries.desc: +"""HTTP request max retry times if failed.""" + +config_max_retries.label: +"""HTTP Request Max Retries""" + +desc_config.desc: +"""Configuration for Apache IoTDB bridge.""" + +desc_config.label: +"""IoTDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +config_parameters_action.desc: +"""TODO""" + +config_parameters_action.label: +"""Action""" + +config_parameters_index.desc: +"""Name of the data stream, index, or index alias to perform the action on. +This parameter is required if a is not specified in the request path.""" + +config_parameters_index.label: +"""_index""" + +config_parameters_id.desc: +"""The document ID. If no ID is specified, a document ID is automatically generated.""" +config_parameters_id.label: +"""_id""" + +config_parameters_require_alias.desc: +"""If true, the action must target an index alias. Defaults to false.""" +config_parameters_require_alias.label: +"""_require_alias""" + +config_parameters_fields.desc: +"""The document source to index. Required for create and index operations.""" +config_parameters_fields.label: +"""fields""" + +config_parameters_doc_as_upsert.desc: +"""Instead of sending a partial doc plus an upsert doc, you can set doc_as_upsert to true +to use the contents of doc as the upsert value.""" +config_parameters_doc_as_upsert.label: +"""doc_as_upsert""" + +config_parameters_upsert.desc: +"""If the document does not already exist, the contents of the upsert element are inserted as a new document.""" +config_parameters_upsert.label: +"""upsert""" + + +action_parameters_data.desc: +"""ElasticSearch action parameter data""" + +action_parameters_data.label: +"""Parameter Data""" + +action_parameters.desc: +"""ElasticSearch action parameters""" + +action_parameters.label: +"""Parameters""" + +} diff --git a/rel/i18n/emqx_bridge_es_connector.hocon b/rel/i18n/emqx_bridge_es_connector.hocon new file mode 100644 index 000000000..f980b3aca --- /dev/null +++ b/rel/i18n/emqx_bridge_es_connector.hocon @@ -0,0 +1,44 @@ +emqx_bridge_es_connector { + +config_authentication.desc: +"""Authentication configuration""" + +config_authentication.label: +"""Authentication""" + +auth_basic.desc: +"""Parameters for basic authentication.""" + +auth_basic.label: +"""Basic auth params""" + +config_auth_basic_username.desc: +"""The username as configured at the ElasticSearch REST interface""" + +config_auth_basic_username.label: + """HTTP Basic Auth Username""" + +config_auth_basic_password.desc: +"""The password as configured at the ElasticSearch REST interface""" + +config_auth_basic_password.label: +"""HTTP Basic Auth Password""" + +config_base_url.desc: +"""The base URL of the external ElasticSearch service's REST interface.""" +config_base_url.label: +"""ElasticSearch REST Service Base URL""" + +config_max_retries.desc: +"""HTTP request max retry times if failed.""" + +config_max_retries.label: +"""HTTP Request Max Retries""" + +desc_config.desc: +"""Configuration for ElasticSearch bridge.""" + +desc_config.label: +"""ElasticSearch Bridge Configuration""" + +} From 4c40e754f4c60367f13fce1f443c494d895a6246 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 13 Jan 2024 15:06:38 +0800 Subject: [PATCH 14/62] chore: forward the async invite to leader node --- apps/emqx/priv/bpapi.versions | 1 + .../src/emqx_mgmt_api_cluster.erl | 20 ++++++++++--------- .../emqx_management/src/emqx_mgmt_cluster.erl | 20 ++++++++++--------- .../test/emqx_mgmt_api_cluster_SUITE.erl | 8 ++++---- rel/i18n/emqx_mgmt_api_cluster.hocon | 5 +++++ 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 2777aec53..9721a7f2f 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -41,6 +41,7 @@ {emqx_mgmt_api_plugins,2}. {emqx_mgmt_cluster,1}. {emqx_mgmt_cluster,2}. +{emqx_mgmt_cluster,3}. {emqx_mgmt_data_backup,1}. {emqx_mgmt_trace,1}. {emqx_mgmt_trace,2}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index d14b372ab..686a0be71 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -78,13 +78,13 @@ schema("/cluster/topology") -> }; schema("/cluster/invitation") -> #{ - 'operationId' => get_invitation_view, + 'operationId' => get_invitation_status, get => #{ - desc => ?DESC(get_invitation_view), + desc => ?DESC(get_invitation_status), tags => [<<"Cluster">>], responses => #{ 200 => ?HOCON( - ?REF(invitation_view), + ?REF(invitation_status), #{desc => <<"Get invitation progress created by async operation">>} ) } @@ -174,7 +174,7 @@ fields(timeout) -> #{desc => <<"Timeout in milliseconds">>, example => <<"15000">>} )} ]; -fields(invitation_view) -> +fields(invitation_status) -> [ {succeed, ?HOCON( @@ -208,7 +208,8 @@ fields(node_invitation_succeed) -> ?HOCON( emqx_utils_calendar:epoch_millisecond(), #{ - desc => <<"The time of the async invitation result is received, millisecond precision epoch">>, + desc => + <<"The time of the async invitation result is received, millisecond precision epoch">>, example => <<"1705044829915">> } )} @@ -224,7 +225,8 @@ fields(node_invitation_in_progress) -> ?HOCON( emqx_utils_calendar:epoch_millisecond(), #{ - desc => <<"The start timestamp of the invitation, millisecond precision epoch">>, + desc => + <<"The start timestamp of the invitation, millisecond precision epoch">>, example => <<"1705044829915">> } )} @@ -319,8 +321,8 @@ invite_node_async(put, #{bindings := #{node := Node0}}) -> }} end. -get_invitation_view(get, _) -> - {200, format_invitation_view(emqx_mgmt_cluster:invitation_view())}. +get_invitation_status(get, _) -> + {200, format_invitation_status(emqx_mgmt_cluster:invitation_status())}. force_leave(delete, #{bindings := #{node := Node0}}) -> Node = ekka_node:parse_name(binary_to_list(Node0)), @@ -344,7 +346,7 @@ connected_replicants() -> error_message(Msg) -> iolist_to_binary(io_lib:format("~p", [Msg])). -format_invitation_view(#{ +format_invitation_status(#{ succeed := Succeed, in_progress := InProgress, failed := Failed diff --git a/apps/emqx_management/src/emqx_mgmt_cluster.erl b/apps/emqx_management/src/emqx_mgmt_cluster.erl index 818e79452..828567776 100644 --- a/apps/emqx_management/src/emqx_mgmt_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_cluster.erl @@ -21,7 +21,7 @@ %% APIs -export([start_link/0]). --export([invite_async/1, invitation_view/0]). +-export([invite_async/1, invitation_status/0]). %% gen_server callbacks -export([ @@ -42,17 +42,19 @@ start_link() -> -spec invite_async(atom()) -> ok | ignore | {error, {already_started, pid()}}. invite_async(Node) -> - JoinTo = node(), + %% Proxy the invitation task to the leader node + JoinTo = mria_membership:leader(), case Node =/= JoinTo of true -> - gen_server:call(?MODULE, {invite_async, Node, JoinTo}, infinity); + gen_server:call({?MODULE, JoinTo}, {invite_async, Node, JoinTo}, infinity); false -> ignore end. --spec invitation_view() -> map(). -invitation_view() -> - gen_server:call(?MODULE, invitation_view, infinity). +-spec invitation_status() -> map(). +invitation_status() -> + Leader = mria_membership:leader(), + gen_server:call({?MODULE, Leader}, invitation_status, infinity). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -71,8 +73,8 @@ handle_call({invite_async, Node, JoinTo}, _From, State) -> WorkerPid -> {reply, {error, {already_started, WorkerPid}}, State} end; -handle_call(invitation_view, _From, State) -> - {reply, state_to_invitation_view(State), State}; +handle_call(invitation_status, _From, State) -> + {reply, state_to_invitation_status(State), State}; handle_call(_Request, _From, State) -> Reply = ok, {reply, Reply, State}. @@ -155,7 +157,7 @@ find_node_name_via_worker_pid(WorkerPid, {Key, Task, I}) -> find_node_name_via_worker_pid(WorkerPid, maps:next(I)) end. -state_to_invitation_view(State) -> +state_to_invitation_status(State) -> History = maps:get(history, State, #{}), {Succ, Failed} = lists:foldl( fun({Node, Task}, {SuccAcc, FailedAcc}) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index d195b04e1..b2658f8fa 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -141,11 +141,11 @@ t_cluster_invite_api_timeout(Config) -> ) end, ?assertMatch( - {400, #{code := 'BAD_REQUEST', message := <<"timeout must be integer">>}}, + {400, #{code := 'BAD_REQUEST', message := <<"timeout must be an integer">>}}, Invite(Core2, not_a_integer_timeout) ), ?assertMatch( - {400, #{code := 'BAD_REQUEST', message := <<"timeout can't less than 5000ms">>}}, + {400, #{code := 'BAD_REQUEST', message := <<"timeout cannot be less than 5000ms">>}}, Invite(Core2, 3000) ), @@ -240,7 +240,7 @@ t_cluster_invite_async(Config) -> %% assert: core2 is in_progress status ?assertMatch( {200, #{in_progress := [#{node := Core2}]}}, - rpc:call(Core1, emqx_mgmt_api_cluster, get_invitation_view, [get, #{}]) + rpc:call(Core1, emqx_mgmt_api_cluster, get_invitation_status, [get, #{}]) ), %% waiting the async invitation_succeed @@ -295,7 +295,7 @@ waiting_the_async_invitation_succeed(Node, TargetNode, N) -> in_progress := InProgress, succeed := Succeed, failed := Failed - }} = rpc:call(Node, emqx_mgmt_api_cluster, get_invitation_view, [get, #{}]), + }} = rpc:call(Node, emqx_mgmt_api_cluster, get_invitation_status, [get, #{}]), case find_node_info_list(TargetNode, InProgress) of error -> case find_node_info_list(TargetNode, Succeed) of diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon index 55ce17a9e..996d8b2fa 100644 --- a/rel/i18n/emqx_mgmt_api_cluster.hocon +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -25,4 +25,9 @@ force_remove_node.desc: force_remove_node.label: """Force leave node from cluster""" +get_invitation_status.desc: +"""Get the execution status of all asynchronous invite node tasks""" +get_invitation_status.label: +"""Get status of all invitation tasks""" + } From ace443fc18263853aed8773339b492e8e552a032 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 12 Jan 2024 14:15:30 +0800 Subject: [PATCH 15/62] refactor: refactor es's action --- .../src/schema/emqx_bridge_v2_schema.erl | 15 +- apps/emqx_bridge_es/.gitignore | 19 - apps/emqx_bridge_es/src/emqx_bridge_es.erl | 361 +++++++----------- .../src/emqx_bridge_es_connector.erl | 263 ++++--------- .../src/emqx_bridge_http_connector.erl | 23 +- .../src/schema/emqx_connector_ee_schema.erl | 2 +- rel/i18n/emqx_bridge_es.hocon | 75 ++-- rel/i18n/emqx_bridge_es_connector.hocon | 5 - scripts/spellcheck/dicts/emqx.txt | 2 + 9 files changed, 275 insertions(+), 490 deletions(-) delete mode 100644 apps/emqx_bridge_es/.gitignore diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 74239ffc0..aa06b547d 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -361,11 +361,16 @@ is_bad_schema(#{type := ?MAP(_, ?R_REF(Module, TypeName))}) -> [] -> false; _ -> - {true, #{ - schema_module => Module, - type_name => TypeName, - missing_fields => MissingFields - }} + %% elasticsearch is new and doesn't have local_topic + case MissingFields of + [local_topic] when Module =:= emqx_bridge_es -> false; + _ -> + {true, #{ + schema_module => Module, + type_name => TypeName, + missing_fields => MissingFields + }} + end end. -endif. diff --git a/apps/emqx_bridge_es/.gitignore b/apps/emqx_bridge_es/.gitignore deleted file mode 100644 index e9bc1c544..000000000 --- a/apps/emqx_bridge_es/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 - _* - .eunit - *.o - *.beam - *.plt - *.swp - *.swo - .erlang.cookie - ebin - log - erl_crash.dump - .rebar - logs - _build - .idea - *.iml - rebar3.crashdump - *~ diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index 57ab648b5..b575f32ed 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -25,15 +25,15 @@ fields(action) -> ?HOCON( ?MAP(action_name, ?R_REF(action_config)), #{ - desc => <<"ElasticSearch Action Config">>, - required => false + required => false, + desc => ?DESC(elasticsearch) } )}; fields(action_config) -> emqx_resource_schema:override( - emqx_bridge_v2_schema:make_producer_action_schema( + emqx_bridge_v2_schema:make_consumer_action_schema( ?HOCON( - ?R_REF(action_parameters), + ?UNION(fun action_union_member_selector/1), #{ required => true, desc => ?DESC("action_parameters") } @@ -54,200 +54,28 @@ fields(action_resource_opts) -> end, emqx_bridge_v2_schema:resource_opts_fields() ); -fields(action_parameters) -> +fields(action_create) -> [ - {target, - ?HOCON( - binary(), - #{ - desc => ?DESC("config_target"), - required => false - } - )}, - {require_alias, - ?HOCON( - boolean(), - #{ - required => false, - default => false, - desc => ?DESC("config_require_alias") - } - )}, - {routing, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_routing") - } - )}, - {wait_for_active_shards, - ?HOCON( - ?UNION([pos_integer(), all]), - #{ - required => false, - desc => ?DESC("config_wait_for_active_shards") - } - )}, - {data, - ?HOCON( - ?ARRAY( - ?UNION( - [ - ?R_REF(create), - ?R_REF(delete), - ?R_REF(index), - ?R_REF(update) - ] - ) - ), - #{ - desc => ?DESC("action_parameters_data") - } - )} - ] ++ - lists:filter( - fun({K, _}) -> - not lists:member(K, [path, method, body, headers, request_timeout]) - end, - emqx_bridge_http_schema:fields("parameters_opts") - ); -fields(Action) when Action =:= create; Action =:= index -> - [ - {action, - ?HOCON( - Action, - #{ - desc => atom_to_binary(Action), - required => true - } - )}, - {'_index', - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_index") - } - )}, - {'_id', - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_id") - } - )}, - {require_alias, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_require_alias") - } - )}, - {fields, - ?HOCON( - binary(), - #{ - required => true, - desc => ?DESC("config_parameters_fields") - } - )} + action(create), + index(), + id(false), + doc(true), + routing(), + require_alias(), + overwrite() + | http_common_opts() ]; -fields(delete) -> +fields(action_delete) -> + [action(delete), index(), id(true), routing() | http_common_opts()]; +fields(action_update) -> [ - {action, - ?HOCON( - delete, - #{ - desc => <<"Delete">>, - required => true - } - )}, - {'_index', - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_index") - } - )}, - {'_id', - ?HOCON( - binary(), - #{ - required => true, - desc => ?DESC("config_parameters_id") - } - )}, - {require_alias, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_require_alias") - } - )} - ]; -fields(update) -> - [ - {action, - ?HOCON( - update, - #{ - desc => <<"Update">>, - required => true - } - )}, - {doc_as_upsert, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_doc_as_upsert") - } - )}, - {upsert, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_upsert") - } - )}, - {'_index', - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_index") - } - )}, - {'_id', - ?HOCON( - binary(), - #{ - required => true, - desc => ?DESC("config_parameters_id") - } - )}, - {require_alias, - ?HOCON( - binary(), - #{ - required => false, - desc => ?DESC("config_parameters_require_alias") - } - )}, - {fields, - ?HOCON( - binary(), - #{ - required => true, - desc => ?DESC("config_parameters_fields") - } - )} + action(update), + index(), + id(true), + doc(true), + routing(), + require_alias() + | http_common_opts() ]; fields("post_bridge_v2") -> emqx_bridge_schema:type_and_name_fields(elasticsearch) ++ fields(action_config); @@ -256,6 +84,111 @@ fields("put_bridge_v2") -> fields("get_bridge_v2") -> emqx_bridge_schema:status_fields() ++ fields("post_bridge_v2"). +action_union_member_selector(all_union_members) -> + [ + ?R_REF(action_create), + ?R_REF(action_delete), + ?R_REF(action_update) + ]; +action_union_member_selector({value, Value}) -> + case Value of + #{<<"action">> := <<"create">>} -> + [?R_REF(action_create)]; + #{<<"action">> := <<"delete">>} -> + [?R_REF(action_delete)]; + #{<<"action">> := <<"update">>} -> + [?R_REF(action_update)]; + _ -> + Expected = "create | delete | update", + throw(#{ + field_name => action, + expected => Expected + }) + end. + +action(Action) -> + {action, + ?HOCON( + Action, + #{ + required => true, + desc => atom_to_binary(Action) + } + )}. + +overwrite() -> + {overwrite, + ?HOCON( + boolean(), + #{ + required => false, + default => true, + desc => ?DESC("config_overwrite") + } + )}. + +index() -> + {index, + ?HOCON( + binary(), + #{ + required => true, + example => <<"${payload.index}">>, + desc => ?DESC("config_parameters_index") + } + )}. + +id(Required) -> + {id, + ?HOCON( + binary(), + #{ + required => Required, + example => <<"${payload.id}">>, + desc => ?DESC("config_parameters_id") + } + )}. + +doc(Required) -> + {doc, + ?HOCON( + binary(), + #{ + required => Required, + example => <<"${payload.doc}">>, + desc => ?DESC("config_parameters_doc") + } + )}. + +http_common_opts() -> + lists:filter( + fun({K, _}) -> + not lists:member(K, [path, method, body, headers, request_timeout]) + end, + emqx_bridge_http_schema:fields("parameters_opts") + ). + +routing() -> + {routing, + ?HOCON( + binary(), + #{ + required => false, + example => <<"${payload.routing}">>, + desc => ?DESC("config_routing") + } + )}. + +require_alias() -> + {require_alias, + ?HOCON( + boolean(), + #{ + required => false, + desc => ?DESC("config_require_alias") + } + )}. + bridge_v2_examples(Method) -> [ #{ @@ -272,34 +205,10 @@ bridge_v2_examples(Method) -> action_values() -> #{ parameters => #{ - target => <<"${target_index}">>, - data => [ - #{ - action => index, - '_index' => <<"${index}">>, - fields => <<"${fields}">>, - require_alias => <<"${require_alias}">> - }, - #{ - action => create, - '_index' => <<"${index}">>, - fields => <<"${fields}">> - }, - #{ - action => delete, - '_index' => <<"${index}">>, - '_id' => <<"${id}">> - }, - #{ - action => update, - '_index' => <<"${index}">>, - '_id' => <<"${id}">>, - fields => <<"${fields}">>, - require_alias => false, - doc_as_upsert => <<"${doc_as_upsert}">>, - upsert => <<"${upsert}">> - } - ] + action => create, + index => <<"${payload.index}">>, + overwrite => true, + doc => <<"${payload.doc}">> } }. @@ -309,4 +218,10 @@ unsupported_opts() -> batch_time ]. +desc(elasticsearch) -> ?DESC(elasticsearch); +desc(action_config) -> ?DESC(action_config); +desc(action_create) -> ?DESC(action_create); +desc(action_delete) -> ?DESC(action_delete); +desc(action_update) -> ?DESC(action_update); +desc(action_resource_opts) -> ?DESC(action_resource_opts); desc(_) -> undefined. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 22509e037..fe86eac56 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -233,7 +233,7 @@ on_get_status(InstanceId, State) -> {ok, pos_integer(), [term()], term()} | {ok, pos_integer(), [term()]} | {error, term()}. -on_query(InstanceId, {ChannelId, Msg} = Req, #{channels := Channels} = State) -> +on_query(InstanceId, {ChannelId, Msg} = Req, State) -> ?tp(elasticsearch_bridge_on_query, #{instance_id => InstanceId}), ?SLOG(debug, #{ msg => "elasticsearch_bridge_on_query_called", @@ -241,21 +241,16 @@ on_query(InstanceId, {ChannelId, Msg} = Req, #{channels := Channels} = State) -> send_message => Req, state => emqx_utils:redact(State) }), - case try_render_message(Req, Channels) of - {ok, Body} -> - handle_response( - emqx_bridge_http_connector:on_query( - InstanceId, {ChannelId, {Msg, Body}}, State - ) - ); - Error -> - Error - end. + handle_response( + emqx_bridge_http_connector:on_query( + InstanceId, {ChannelId, Msg}, State + ) + ). -spec on_query_async(manager_id(), tuple(), {function(), [term()]}, state()) -> {ok, pid()} | {error, empty_request}. on_query_async( - InstanceId, {ChannelId, Msg} = Req, ReplyFunAndArgs0, #{channels := Channels} = State + InstanceId, {ChannelId, Msg} = Req, ReplyFunAndArgs0, State ) -> ?tp(elasticsearch_bridge_on_query_async, #{instance_id => InstanceId}), ?SLOG(debug, #{ @@ -264,22 +259,17 @@ on_query_async( send_message => Req, state => emqx_utils:redact(State) }), - case try_render_message(Req, Channels) of - {ok, Payload} -> - ReplyFunAndArgs = - { - fun(Result) -> - Response = handle_response(Result), - emqx_resource:apply_reply_fun(ReplyFunAndArgs0, Response) - end, - [] - }, - emqx_bridge_http_connector:on_query_async( - InstanceId, {ChannelId, {Msg, Payload}}, ReplyFunAndArgs, State - ); - Error -> - Error - end. + ReplyFunAndArgs = + { + fun(Result) -> + Response = handle_response(Result), + emqx_resource:apply_reply_fun(ReplyFunAndArgs0, Response) + end, + [] + }, + emqx_bridge_http_connector:on_query_async( + InstanceId, {ChannelId, Msg}, ReplyFunAndArgs, State + ). on_add_channel( InstanceId, @@ -291,19 +281,17 @@ on_add_channel( true -> {error, already_exists}; _ -> - #{data := Data} = Parameter, - Parameter1 = Parameter#{path => path(Parameter), method => <<"post">>}, + Parameter1 = Parameter#{ + path => path(Parameter), + method => method(Parameter), + body => get_body_template(Parameter) + }, {ok, State} = emqx_bridge_http_connector:on_add_channel( InstanceId, State0, ChannelId, #{parameters => Parameter1} ), - case preproc_data_template(Data) of - [] -> - {error, invalid_data}; - DataTemplate -> - Channel = Parameter1#{data => DataTemplate}, - Channels2 = Channels#{ChannelId => Channel}, - {ok, State#{channels => Channels2}} - end + Channel = Parameter1, + Channels2 = Channels#{ChannelId => Channel}, + {ok, State#{channels => Channels2}} end. on_remove_channel(InstanceId, #{channels := Channels} = OldState0, ChannelId) -> @@ -325,124 +313,55 @@ on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> %%-------------------------------------------------------------------- %% Internal Functions %%-------------------------------------------------------------------- -path(Param) -> - Target = maps:get(target, Param, undefined), - QString0 = maps:fold( - fun(K, V, Acc) -> - [[atom_to_list(K), "=", to_str(V)] | Acc] +%% delete DELETE //_doc/<_id> +path(#{action := delete, id := Id, index := Index} = Action) -> + BasePath = ["/", Index, "/_doc/", Id], + Qs = add_query_string([routing], Action), + BasePath ++ Qs; +%% update POST //_update/<_id> +path(#{action := update, id := Id, index := Index} = Action) -> + BasePath = ["/", Index, "/_update/", Id], + Qs = add_query_string([routing, require_alias], Action), + BasePath ++ Qs; +%% create with id //_doc/_id +path(#{action := create, index := Index, id := Id} = Action) -> + BasePath = ["/", Index, "/_doc/", Id], + Qs = + case maps:get(overwrite, Action, true) of + true -> + add_query_string([routing, require_alias], Action); + false -> + Action1 = Action#{op_type => "create"}, + add_query_string([routing, require_alias, op_type], Action1) end, - [["_source=false"], ["filter_path=items.*.error"]], - maps:with([require_alias, routing, wait_for_active_shards], Param) - ), - QString = "?" ++ lists:join("&", QString0), - target(Target) ++ QString. + BasePath ++ Qs; +%% create without id POST //_doc/ +path(#{action := create, index := Index} = Action) -> + BasePath = ["/", Index, "/_doc/"], + Qs = add_query_string([routing, require_alias], Action), + BasePath ++ Qs. -target(undefined) -> "/_bulk"; -target(Str) -> "/" ++ binary_to_list(Str) ++ "/_bulk". +method(#{action := create}) -> <<"POST">>; +method(#{action := delete}) -> <<"DELETE">>; +method(#{action := update}) -> <<"POST">>. + +add_query_string(Keys, Param0) -> + Param1 = maps:with(Keys, Param0), + FoldFun = fun(K, V, Acc) -> [[atom_to_list(K), "=", to_str(V)] | Acc] end, + case maps:fold(FoldFun, [], Param1) of + "" -> ""; + QString -> "?" ++ lists:join("&", QString) + end. to_str(List) when is_list(List) -> List; to_str(false) -> "false"; to_str(true) -> "true"; to_str(Atom) when is_atom(Atom) -> atom_to_list(Atom). -proc_data(DataList, Msg) when is_list(DataList) -> - [ - begin - proc_data(Data, Msg) - end - || Data <- DataList - ]; -proc_data( - #{ - action := Action, - '_index' := IndexT, - '_id' := IdT, - require_alias := RequiredAliasT, - fields := FieldsT - }, - Msg -) when Action =:= create; Action =:= index -> - [ - emqx_utils_json:encode( - #{ - Action => filter([ - {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, - {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, - {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)} - ]) - } - ), - "\n", - emqx_placeholder:proc_tmpl(FieldsT, Msg), - "\n" - ]; -proc_data( - #{ - action := delete, - '_index' := IndexT, - '_id' := IdT, - require_alias := RequiredAliasT - }, - Msg -) -> - [ - emqx_utils_json:encode( - #{ - delete => filter([ - {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, - {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, - {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)} - ]) - } - ), - "\n" - ]; -proc_data( - #{ - action := update, - '_index' := IndexT, - '_id' := IdT, - require_alias := RequiredAliasT, - doc_as_upsert := DocAsUpsert, - upsert := Upsert, - fields := FieldsT - }, - Msg -) -> - [ - emqx_utils_json:encode( - #{ - update => filter([ - {'_index', emqx_placeholder:proc_tmpl(IndexT, Msg)}, - {'_id', emqx_placeholder:proc_tmpl(IdT, Msg)}, - {required_alias, emqx_placeholder:proc_tmpl(RequiredAliasT, Msg)}, - {doc_as_upsert, emqx_placeholder:proc_tmpl(DocAsUpsert, Msg)}, - {upsert, emqx_placeholder:proc_tmpl(Upsert, Msg)} - ]) - } - ), - "\n{\"doc\":", - emqx_placeholder:proc_tmpl(FieldsT, Msg), - "}\n" - ]. - -filter(List) -> - Fun = fun - ({_K, V}) when V =:= undefined; V =:= <<"undefined">>; V =:= "undefined" -> - false; - ({_K, V}) when V =:= ""; V =:= <<>> -> - false; - ({_K, V}) when V =:= "false" -> {true, false}; - ({_K, V}) when V =:= "true" -> {true, true}; - ({_K, _V}) -> - true - end, - maps:from_list(lists:filtermap(Fun, List)). - -handle_response({ok, 200, _Headers, Body} = Resp) -> - eval_response_body(Body, Resp); -handle_response({ok, 200, Body} = Resp) -> - eval_response_body(Body, Resp); +handle_response({ok, Code, _Headers, _Body} = Resp) when Code =:= 200; Code =:= 201 -> + Resp; +handle_response({ok, Code, _Body} = Resp) when Code =:= 200; Code =:= 201 -> + Resp; handle_response({ok, Code, _Headers, Body}) -> {error, #{code => Code, body => Body}}; handle_response({ok, Code, Body}) -> @@ -450,49 +369,5 @@ handle_response({ok, Code, Body}) -> handle_response({error, _} = Error) -> Error. -eval_response_body(<<"{}">>, Resp) -> Resp; -eval_response_body(Body, _Resp) -> {error, emqx_utils_json:decode(Body)}. - -preproc_data_template(DataList) when is_list(DataList) -> - [ - begin - preproc_data_template(Data) - end - || Data <- DataList - ]; -preproc_data_template(#{action := create} = Data) -> - Index = maps:get('_index', Data, ""), - Id = maps:get('_id', Data, ""), - RequiredAlias = maps:get(require_alias, Data, ""), - Fields = maps:get(fields, Data, ""), - #{ - action => create, - '_index' => emqx_placeholder:preproc_tmpl(Index), - '_id' => emqx_placeholder:preproc_tmpl(Id), - require_alias => emqx_placeholder:preproc_tmpl(RequiredAlias), - fields => emqx_placeholder:preproc_tmpl(Fields) - }; -preproc_data_template(#{action := index} = Data) -> - Data1 = preproc_data_template(Data#{action => create}), - Data1#{action => index}; -preproc_data_template(#{action := delete} = Data) -> - Data1 = preproc_data_template(Data#{action => create}), - Data2 = Data1#{action => delete}, - maps:remove(fields, Data2); -preproc_data_template(#{action := update} = Data) -> - Data1 = preproc_data_template(Data#{action => index}), - DocAsUpsert = maps:get(doc_as_upsert, Data, ""), - Upsert = maps:get(upsert, Data, ""), - Data1#{ - action => update, - doc_as_upsert => emqx_placeholder:preproc_tmpl(DocAsUpsert), - upsert => emqx_placeholder:preproc_tmpl(Upsert) - }. - -try_render_message({ChannelId, Msg}, Channels) -> - case maps:find(ChannelId, Channels) of - {ok, #{data := Data}} -> - {ok, proc_data(Data, Msg)}; - _ -> - {error, {unrecoverable_error, {invalid_channel_id, ChannelId}}} - end. +get_body_template(#{doc := Doc}) -> Doc; +get_body_template(_) -> undefined. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index f00ae8523..8f54694e9 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -317,7 +317,7 @@ on_query(InstId, {send_message, Msg}, State) -> %% BridgeV2 entrypoint on_query( InstId, - {ActionId, MsgAndBody}, + {ActionId, Msg}, State = #{installed_actions := InstalledActions} ) when is_binary(ActionId) -> case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of @@ -334,10 +334,10 @@ on_query( body := Body, headers := Headers, request_timeout := Timeout - } = process_request_and_action(Request, ActionState, MsgAndBody), + } = process_request_and_action(Request, ActionState, Msg), %% bridge buffer worker has retry, do not let ehttpc retry Retry = 2, - ClientId = clientid(MsgAndBody), + ClientId = clientid(Msg), on_query( InstId, {ClientId, Method, {Path, Headers, Body}, Timeout, Retry}, @@ -430,7 +430,7 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> %% BridgeV2 entrypoint on_query_async( InstId, - {ActionId, MsgAndBody}, + {ActionId, Msg}, ReplyFunAndArgs, State = #{installed_actions := InstalledActions} ) when is_binary(ActionId) -> @@ -448,8 +448,8 @@ on_query_async( body := Body, headers := Headers, request_timeout := Timeout - } = process_request_and_action(Request, ActionState, MsgAndBody), - ClientId = clientid(MsgAndBody), + } = process_request_and_action(Request, ActionState, Msg), + ClientId = clientid(Msg), on_query_async( InstId, {ClientId, Method, {Path, Headers, Body}, Timeout}, @@ -629,7 +629,7 @@ maybe_parse_template(Key, Conf) -> parse_template(String) -> emqx_template:parse(String). -process_request_and_action(Request, ActionState, {Msg, Body}) -> +process_request_and_action(Request, ActionState, Msg) -> MethodTemplate = maps:get(method, ActionState), Method = make_method(render_template_string(MethodTemplate, Msg)), PathPrefix = unicode:characters_to_list(render_template(maps:get(path, Request), Msg)), @@ -647,17 +647,15 @@ process_request_and_action(Request, ActionState, {Msg, Body}) -> render_headers(HeadersTemplate1, Msg), render_headers(HeadersTemplate2, Msg) ), + BodyTemplate = maps:get(body, ActionState), + Body = render_request_body(BodyTemplate, Msg), #{ method => Method, path => Path, body => Body, headers => Headers, request_timeout => maps:get(request_timeout, ActionState) - }; -process_request_and_action(Request, ActionState, Msg) -> - BodyTemplate = maps:get(body, ActionState), - Body = render_request_body(BodyTemplate, Msg), - process_request_and_action(Request, ActionState, {Msg, Body}). + }. merge_proplist(Proplist1, Proplist2) -> lists:foldl( @@ -877,7 +875,6 @@ redact_request({Path, Headers}) -> redact_request({Path, Headers, _Body}) -> {Path, Headers, <<"******">>}. -clientid({Msg, _Body}) -> clientid(Msg); clientid(Msg) -> maps:get(clientid, Msg, undefined). -ifdef(TEST). diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 822e8429b..655892d88 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -190,7 +190,7 @@ connector_structs() -> mk( hoconsc:map(name, ref(emqx_bridge_es_connector, config)), #{ - desc => <<"Elastis Search Connector Config">>, + desc => <<"ElasticSearch Connector Config">>, required => false } )} diff --git a/rel/i18n/emqx_bridge_es.hocon b/rel/i18n/emqx_bridge_es.hocon index 78299c4ee..62778a712 100644 --- a/rel/i18n/emqx_bridge_es.hocon +++ b/rel/i18n/emqx_bridge_es.hocon @@ -1,5 +1,10 @@ emqx_bridge_es { +elasticsearch.desc: +"""Elasticsearch Bridge""" +elasticsearch.label: +"""ElasticSearch""" + config_enable.desc: """Enable or disable this bridge""" @@ -74,15 +79,9 @@ desc_name.desc: desc_name.label: """Bridge Name""" -config_parameters_action.desc: -"""TODO""" - -config_parameters_action.label: -"""Action""" - config_parameters_index.desc: -"""Name of the data stream, index, or index alias to perform the action on. -This parameter is required if a is not specified in the request path.""" +"""Name of index, or index alias to perform the action on. +This parameter is required.""" config_parameters_index.label: """_index""" @@ -97,28 +96,10 @@ config_parameters_require_alias.desc: config_parameters_require_alias.label: """_require_alias""" -config_parameters_fields.desc: -"""The document source to index. Required for create and index operations.""" -config_parameters_fields.label: -"""fields""" - -config_parameters_doc_as_upsert.desc: -"""Instead of sending a partial doc plus an upsert doc, you can set doc_as_upsert to true -to use the contents of doc as the upsert value.""" -config_parameters_doc_as_upsert.label: -"""doc_as_upsert""" - -config_parameters_upsert.desc: -"""If the document does not already exist, the contents of the upsert element are inserted as a new document.""" -config_parameters_upsert.label: -"""upsert""" - - -action_parameters_data.desc: -"""ElasticSearch action parameter data""" - -action_parameters_data.label: -"""Parameter Data""" +config_parameters_doc.desc: +"""JSON document""" +config_parameters_doc.label: +"""doc""" action_parameters.desc: """ElasticSearch action parameters""" @@ -126,4 +107,38 @@ action_parameters.desc: action_parameters.label: """Parameters""" +config_overwrite.desc: +"""Set to false If a document with the specified _id already exists(conflict), the operation will fail.""" + +config_overwrite.label: +"""overwrite""" + +action_config.desc: +"""ElasticSearch Action Configuration""" +action_config.label: +"""ElasticSearch Action Config""" + +action_create.desc: +"""Adds a JSON document to the specified index and makes it searchable. +If the target is an index and the document already exists, +the request updates the document and increments its version.""" +action_create.label: +"""Create Doc""" + +action_delete.desc: +"""Removes a JSON document from the specified index.""" +action_delete.label: +"""Delete Doc""" + +action_update.desc: +"""Updates a document using the specified doc.""" +action_update.label: +"""Update Doc""" + +action_resource_opts.desc: +"""Resource options.""" + +action_resource_opts.label: +"""Resource Options""" + } diff --git a/rel/i18n/emqx_bridge_es_connector.hocon b/rel/i18n/emqx_bridge_es_connector.hocon index f980b3aca..ddd53e0fc 100644 --- a/rel/i18n/emqx_bridge_es_connector.hocon +++ b/rel/i18n/emqx_bridge_es_connector.hocon @@ -24,11 +24,6 @@ config_auth_basic_password.desc: config_auth_basic_password.label: """HTTP Basic Auth Password""" -config_base_url.desc: -"""The base URL of the external ElasticSearch service's REST interface.""" -config_base_url.label: -"""ElasticSearch REST Service Base URL""" - config_max_retries.desc: """HTTP request max retry times if failed.""" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 401df33a6..1d98d82db 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -297,3 +297,5 @@ Syskeeper msacc now_us ns +elasticsearch +ElasticSearch From c77837a9ea0983b74bb1f261f8b8d929fc7b4f14 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 15 Jan 2024 08:24:33 +0100 Subject: [PATCH 16/62] feat(quic): support reload with new binding port --- apps/emqx/rebar.config.script | 2 +- apps/emqx/src/emqx_listeners.erl | 51 ++++++++++++++++++++++---------- changes/feat-12325.en.md | 1 + mix.exs | 2 +- rebar.config.erl | 2 +- 5 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 changes/feat-12325.en.md diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 8c107d7e5..5f5ce18ba 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.311"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.312"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 74417f3b7..95815dacd 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -469,26 +469,23 @@ do_update_listener(Type, Name, OldConf, NewConf) when ok = ranch:set_protocol_options(Id, WsOpts), %% No-op if the listener was not suspended. ranch:resume_listener(Id); -do_update_listener(quic = Type, Name, _OldConf, NewConf) -> +do_update_listener(quic = Type, Name, OldConf, NewConf) -> case quicer:listener(listener_id(Type, Name)) of {ok, ListenerPid} -> - case quicer_listener:reload(ListenerPid, to_quicer_listener_opts(NewConf)) of + ListenOn = quic_listen_on(maps:get(bind, NewConf)), + case quicer_listener:reload(ListenerPid, ListenOn, to_quicer_listener_opts(NewConf)) of ok -> ok; - {error, _} = Error -> - %% @TODO: prefer: case quicer_listener:reload(ListenerPid, to_quicer_listener_opts(OldConf)) of - case quicer_listener:unlock(ListenerPid, 3000) of + Error -> + case + quic_listener_conf_rollback( + ListenerPid, to_quicer_listener_opts(OldConf), Error + ) + of ok -> - ?ELOG("Failed to reload QUIC listener ~p, but Rollback success\n", [ - Error - ]), {skip, Error}; - RestoreErr -> - ?ELOG( - "Failed to reload QUIC listener ~p, and Rollback failed as well\n", - [Error] - ), - {error, {rollback_fail, RestoreErr}} + E -> + E end end; E -> @@ -991,7 +988,7 @@ quic_listen_on(Bind) -> Port end. --spec to_quicer_listener_opts(map()) -> quicer:listener_opts(). +-spec to_quicer_listener_opts(map()) -> map(). to_quicer_listener_opts(Opts) -> DefAcceptors = erlang:system_info(schedulers_online) * 8, SSLOpts = maps:from_list(ssl_opts(Opts)), @@ -1018,3 +1015,27 @@ to_quicer_listener_opts(Opts) -> ), %% @NOTE: Optional options take precedence over required options maps:merge(Opts2, optional_quic_listener_opts(Opts)). + +-spec quic_listener_conf_rollback( + pid(), + map(), + Error :: {error, _, _} | {error, _} +) -> ok | {error, any()}. +quic_listener_conf_rollback(ListenerPid, #{bind := Bind} = Conf, Error) -> + ListenOn = quic_listen_on(Bind), + case quicer_listener:reload(ListenerPid, ListenOn, Conf) of + ok -> + ?ELOG( + "Failed to reload QUIC listener ~p, but Rollback success\n", + [ + Error + ] + ), + ok; + RestoreErr -> + ?ELOG( + "Failed to reload QUIC listener ~p, and Rollback failed as well\n", + [Error] + ), + {error, {rollback_fail, RestoreErr}} + end. diff --git a/changes/feat-12325.en.md b/changes/feat-12325.en.md new file mode 100644 index 000000000..cc5c5dd7e --- /dev/null +++ b/changes/feat-12325.en.md @@ -0,0 +1 @@ +QUIC listener supports reload the listener binding without disrupting existing connections. diff --git a/mix.exs b/mix.exs index 5ef6fef36..7389cb0f9 100644 --- a/mix.exs +++ b/mix.exs @@ -795,7 +795,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.311", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.312", override: true}], else: [] end diff --git a/rebar.config.erl b/rebar.config.erl index e77b799f8..e374959dc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -36,7 +36,7 @@ assert_otp() -> end. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.311"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.312"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.12"}}}. From d2d3ddb72ae81a0a936cdc9afb4f0d9ab10138e4 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 15 Jan 2024 11:42:45 +0100 Subject: [PATCH 17/62] test(quic): listener port updates --- apps/emqx/test/emqx_listeners_SUITE.erl | 33 ++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 542016131..2d2a13e31 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -444,14 +444,45 @@ t_quic_update_opts(Config) -> | ClientSSLOpts ]), + %% Change the listener port + NewPort = emqx_common_test_helpers:select_free_port(ListenerType), + {ok, _} = emqx:update_config( + [listeners, ListenerType, updated], + {update, #{ + <<"bind">> => format_bind({Host, NewPort}) + }} + ), + + %% Connect to old port fail + ?assertExceptionOneOf( + {exit, _}, + {error, _}, + ConnectFun(Host, Port, [ + {cacertfile, filename:join(PrivDir, "ca-next.pem")}, + {certfile, filename:join(PrivDir, "client.pem")}, + {keyfile, filename:join(PrivDir, "client.key")} + | ClientSSLOpts + ]) + ), + + %% Connect to new port successfully. + C4 = ConnectFun(Host, NewPort, [ + {cacertfile, filename:join(PrivDir, "ca-next.pem")}, + {certfile, filename:join(PrivDir, "client.pem")}, + {keyfile, filename:join(PrivDir, "client.key")} + | ClientSSLOpts + ]), + %% Both pre- and post-update clients should be alive. ?assertEqual(pong, emqtt:ping(C1)), ?assertEqual(pong, emqtt:ping(C2)), ?assertEqual(pong, emqtt:ping(C3)), + ?assertEqual(pong, emqtt:ping(C4)), ok = emqtt:stop(C1), ok = emqtt:stop(C2), - ok = emqtt:stop(C3) + ok = emqtt:stop(C3), + ok = emqtt:stop(C4) end). t_quic_update_opts_fail(Config) -> From 9441cf080b27a06cf83e3fca13de6da7427e99d9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 15 Jan 2024 18:48:17 +0800 Subject: [PATCH 18/62] chore: apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_mgmt_api_cluster.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_mgmt_api_cluster.hocon b/rel/i18n/emqx_mgmt_api_cluster.hocon index 996d8b2fa..24de515d9 100644 --- a/rel/i18n/emqx_mgmt_api_cluster.hocon +++ b/rel/i18n/emqx_mgmt_api_cluster.hocon @@ -26,8 +26,8 @@ force_remove_node.label: """Force leave node from cluster""" get_invitation_status.desc: -"""Get the execution status of all asynchronous invite node tasks""" +"""Get the execution status of all asynchronous invite status per node""" get_invitation_status.label: -"""Get status of all invitation tasks""" +"""Get invitation statuses""" } From 50ce2384d9b4b30461731c2d82714cd9d8278426 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 15 Jan 2024 20:29:58 +0800 Subject: [PATCH 19/62] fix(iotdb): ensure the `data` field is `required` --- apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 781eae4b6..83594d53d 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -101,7 +101,8 @@ fields(action_parameters) -> mk( array(ref(?MODULE, action_parameters_data)), #{ - desc => ?DESC("action_parameters_data") + desc => ?DESC("action_parameters_data"), + required => true } )} ] ++ From f7bda457fa8c6c8ec5076cda541bcf5dde7b9393 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 15 Jan 2024 17:57:29 +0100 Subject: [PATCH 20/62] chore: add changelog entry --- changes/ce/feat-12329.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/feat-12329.en.md diff --git a/changes/ce/feat-12329.en.md b/changes/ce/feat-12329.en.md new file mode 100644 index 000000000..c6b8a2b21 --- /dev/null +++ b/changes/ce/feat-12329.en.md @@ -0,0 +1,2 @@ +Add `broker.routing.batch_sync` configuration that enables separate process pool used to synchronize subscriptions with the global routing table in a batched manner. +It's especially useful on nodes interconnected with the cluster through links with non-negligible latency, but might help in other scenarios by ensuring that the broker pool has less chance being overloaded. From ab66986f16fd69eaa0239ec2de96d9520cc21f52 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Jan 2024 13:29:49 +0100 Subject: [PATCH 21/62] feat: add 'tags' field for action and connector --- apps/emqx/src/emqx_schema.erl | 13 ++++++++++++- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 1 + .../src/schema/emqx_bridge_v2_schema.erl | 2 ++ .../src/emqx_bridge_azure_event_hub.erl | 1 + .../src/emqx_bridge_confluent_producer.erl | 1 + .../src/emqx_bridge_http_schema.erl | 2 ++ apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl | 1 + .../src/emqx_bridge_syskeeper.erl | 1 + .../src/schema/emqx_connector_schema.erl | 1 + changes/ce/feat-12333.en.md | 3 +++ rel/i18n/emqx_schema.hocon | 5 +++++ 11 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 changes/ce/feat-12333.en.md diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 1dd0a55ed..66520df75 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -167,7 +167,8 @@ client_ssl_opts_schema/1, ciphers_schema/1, tls_versions_schema/1, - description_schema/0 + description_schema/0, + tags_schema/0 ]). -export([password_converter/2, bin_str_converter/2]). -export([authz_fields/0]). @@ -3825,3 +3826,13 @@ description_schema() -> importance => ?IMPORTANCE_LOW } ). + +tags_schema() -> + sc( + hoconsc:array(string()), + #{ + desc => ?DESC(resource_tags), + required => false, + importance => ?IMPORTANCE_LOW + } + ). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 6a1cb7fbc..ee2dbafa7 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -126,6 +126,7 @@ common_bridge_fields() -> default => true } )}, + {tags, emqx_schema:tags_schema()}, %% Create v2 connector then usr v1 /bridges_probe api to test connector %% /bridges_probe should pass through v2 connector's description. {description, emqx_schema:description_schema()} diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index e144f332d..514eb6988 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -270,6 +270,7 @@ examples(Method) -> top_level_common_action_keys() -> [ <<"connector">>, + <<"tags">>, <<"description">>, <<"enable">>, <<"local_topic">>, @@ -301,6 +302,7 @@ make_consumer_action_schema(ActionParametersRef, Opts) -> mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()}, {parameters, ActionParametersRef}, {resource_opts, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index a63249fa2..4ded55910 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -134,6 +134,7 @@ fields(actions) -> mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ], override_documentations(Fields); diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl index dcae031eb..9c647c62d 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl @@ -121,6 +121,7 @@ fields(actions) -> mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ], override_documentations(Fields); diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index a4d956d78..009eb75e6 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -76,6 +76,7 @@ fields("http_action") -> mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()}, %% Note: there's an implicit convention in `emqx_bridge' that, %% for egress bridges with this config, the published messages @@ -175,6 +176,7 @@ basic_config() -> default => true } )}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ] ++ connector_opts(). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index d74ff40a1..061543b2b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -297,6 +297,7 @@ fields(kafka_producer_action) -> mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ] ++ producer_opts(action); fields(kafka_consumer) -> diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl index 091b84196..7d506b9c8 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl @@ -86,6 +86,7 @@ fields(action) -> fields(config) -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()}, {connector, mk(binary(), #{ diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index b043ebacd..ad28d0251 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -503,6 +503,7 @@ api_fields("put_connector", _Type, Fields) -> common_fields() -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ]. diff --git a/changes/ce/feat-12333.en.md b/changes/ce/feat-12333.en.md new file mode 100644 index 000000000..c29edca8b --- /dev/null +++ b/changes/ce/feat-12333.en.md @@ -0,0 +1,3 @@ +Add 'tags' field for actions and connectors + +Similar to 'description' field (which is a free text annotation), 'tags' can be used to annotate actions and connectors for filtering/grouping. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index fe315b5d7..af4251328 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1570,6 +1570,11 @@ description.label: description.desc: """Descriptive text.""" +resource_tags.label: +"""Tags""" +resource_tags.desc: +"""Tags to annotate this config entry.""" + session_persistence_enable.desc: """Use durable storage for client sessions persistence. If enabled, sessions configured to outlive client connections, along with their corresponding messages, will be durably stored and survive broker downtime.""" From 1fe1a62fe2785fedab162654376c99f0adec9900 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Jan 2024 14:00:34 +0100 Subject: [PATCH 22/62] test: fix already exported function warning --- apps/emqx/test/emqx_common_test_helpers.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d9c9470eb..3ffcb1a6a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -50,8 +50,6 @@ client_ssl/1, client_mtls/0, client_mtls/1, - ssl_verify_fun_allow_any_host/0, - ssl_verify_fun_allow_any_host_impl/3, ensure_mnesia_stopped/0, ensure_quic_listener/2, ensure_quic_listener/3, From 541525c50f37d5198b7d3d0fec83b935201b872a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Jan 2024 14:40:53 +0100 Subject: [PATCH 23/62] fix(authz): fix authz result logs prior to this fix, it's always the default authz result logged at warning level --- apps/emqx/src/emqx_access_control.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 983d78a64..6e8f9b181 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -153,7 +153,7 @@ do_authorize(ClientInfo, Action, Topic) -> case run_hooks('client.authorize', [ClientInfo, Action, Topic], Default) of AuthzResult = #{result := Result} when Result == allow; Result == deny -> From = maps:get(from, AuthzResult, unknown), - ok = log_result(ClientInfo, Topic, Action, From, NoMatch), + ok = log_result(ClientInfo, Topic, Action, From, Result), emqx_hooks:run( 'client.check_authz_complete', [ClientInfo, Action, Topic, Result, From] From 996a851cf67347fd863c5c1800d0cb04a973b32a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Jan 2024 14:42:10 +0100 Subject: [PATCH 24/62] chore: format username as string in log formatter --- apps/emqx/src/emqx_logger_textfmt.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_logger_textfmt.erl b/apps/emqx/src/emqx_logger_textfmt.erl index 2e8718c37..08e6c8d56 100644 --- a/apps/emqx/src/emqx_logger_textfmt.erl +++ b/apps/emqx/src/emqx_logger_textfmt.erl @@ -48,12 +48,17 @@ is_list_report_acceptable(_) -> enrich_report(ReportRaw, Meta) -> %% clientid and peername always in emqx_conn's process metadata. - %% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2 + %% topic and username can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2 Topic = case maps:get(topic, Meta, undefined) of undefined -> maps:get(topic, ReportRaw, undefined); Topic0 -> Topic0 end, + Username = + case maps:get(username, Meta, undefined) of + undefined -> maps:get(username, ReportRaw, undefined); + Username0 -> Username0 + end, ClientId = maps:get(clientid, Meta, undefined), Peer = maps:get(peername, Meta, undefined), MFA = emqx_utils:format_mfal(Meta), @@ -64,8 +69,9 @@ enrich_report(ReportRaw, Meta) -> ({_, undefined}, Acc) -> Acc; (Item, Acc) -> [Item | Acc] end, - maps:to_list(maps:without([topic, msg, clientid], ReportRaw)), + maps:to_list(maps:without([topic, msg, clientid, username], ReportRaw)), [ + {username, try_format_unicode(Username)}, {topic, try_format_unicode(Topic)}, {clientid, try_format_unicode(ClientId)}, {peername, Peer}, From f199a0f24a6a75584e337016e427c8dfd4d5a483 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 1 Dec 2023 10:20:58 +0100 Subject: [PATCH 25/62] feat: refactor MQTT bridge to source, action, and connector This commit: * refactors the MQTT V1 bridge into connector, source and action * Extends the compatibility layer so it works for sources * Fixes the MQTT bridge test suite so that all test cases passes We still need to add a HTTP API handling sources. Also, we still need to add HTTP API example schemes and examples for the MQTT connector/action/source. We should also make sure that we handle the corner cases of the MQTT V1 bridge automatic upgrade downgrade in a sufficiently good way: * An error is currently thrown when converting an MQTT V1 bridge without egress or ingress config. * If there is a source and action with the same name we will currently throw an error in the compatibility layer. * We will also throw an error when converting an MQTT V1 bridge with both ingress and egress. The above is probably the right thing to do but we have to make sure that we return a reasonable error to the user when this happens. (partly) Fixes: https://emqx.atlassian.net/browse/EMQX-11489 --- apps/emqx_bridge/src/emqx_action_info.erl | 76 ++- apps/emqx_bridge/src/emqx_bridge.erl | 8 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 14 +- apps/emqx_bridge/src/emqx_bridge_lib.erl | 3 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 543 +++++++++++++----- .../src/schema/emqx_bridge_v2_schema.erl | 42 +- .../src/emqx_bridge_mqtt_connector.erl | 404 ++++++++----- .../src/emqx_bridge_mqtt_connector_schema.erl | 69 ++- .../src/emqx_bridge_mqtt_egress.erl | 84 --- .../src/emqx_bridge_mqtt_ingress.erl | 293 ++++++---- .../emqx_bridge_mqtt_pubsub_action_info.erl | 221 +++++++ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 129 +++++ .../test/emqx_bridge_mqtt_SUITE.erl | 42 +- .../src/emqx_connector_resource.erl | 23 +- .../src/schema/emqx_connector_schema.erl | 161 ++++-- 15 files changed, 1536 insertions(+), 576 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl create mode 100644 apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index 5ce60fe6c..d80050191 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -26,7 +26,10 @@ bridge_v1_type_to_action_type/1, bridge_v1_type_name/1, is_action_type/1, - registered_schema_modules/0, + is_source/1, + is_action/1, + registered_schema_modules_actions/0, + registered_schema_modules_sources/0, connector_action_config_to_bridge_v1_config/2, connector_action_config_to_bridge_v1_config/3, bridge_v1_config_to_connector_config/2, @@ -51,19 +54,26 @@ ConnectorConfig :: map(), ActionConfig :: map() ) -> map(). %% Define this if the automatic config upgrade is not enough for the connector. --callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> map(). +-callback bridge_v1_config_to_connector_config(BridgeV1Config :: map()) -> + map() | {ConnectorTypeName :: atom(), map()}. %% Define this if the automatic config upgrade is not enough for the bridge. %% If you want to make use of the automatic config upgrade, you can call %% emqx_action_info:transform_bridge_v1_config_to_action_config/4 in your %% implementation and do some adjustments on the result. -callback bridge_v1_config_to_action_config(BridgeV1Config :: map(), ConnectorName :: binary()) -> - map(). + map() | {source | action, ActionTypeName :: atom(), map()} | 'none'. +-callback is_source() -> + boolean(). +-callback is_action() -> + boolean(). -optional_callbacks([ bridge_v1_type_name/0, connector_action_config_to_bridge_v1_config/2, bridge_v1_config_to_connector_config/1, - bridge_v1_config_to_action_config/2 + bridge_v1_config_to_action_config/2, + is_source/0, + is_action/0 ]). %% ==================================================================== @@ -96,7 +106,10 @@ hard_coded_action_info_modules_ee() -> -endif. hard_coded_action_info_modules_common() -> - [emqx_bridge_http_action_info]. + [ + emqx_bridge_http_action_info, + emqx_bridge_mqtt_pubsub_action_info + ]. hard_coded_action_info_modules() -> hard_coded_action_info_modules_common() ++ hard_coded_action_info_modules_ee(). @@ -178,10 +191,33 @@ is_action_type(Type) -> _ -> true end. -registered_schema_modules() -> +%% Returns true if the action is an ingress action, false otherwise. +is_source(Bin) when is_binary(Bin) -> + is_source(binary_to_existing_atom(Bin)); +is_source(Type) -> + ActionInfoMap = info_map(), + IsSourceMap = maps:get(is_source, ActionInfoMap), + maps:get(Type, IsSourceMap, false). + +%% Returns true if the action is an egress action, false otherwise. +is_action(Bin) when is_binary(Bin) -> + is_action(binary_to_existing_atom(Bin)); +is_action(Type) -> + ActionInfoMap = info_map(), + IsActionMap = maps:get(is_action, ActionInfoMap), + maps:get(Type, IsActionMap, true). + +registered_schema_modules_actions() -> InfoMap = info_map(), Schemas = maps:get(action_type_to_schema_module, InfoMap), - maps:to_list(Schemas). + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_action(Type)]. + +registered_schema_modules_sources() -> + InfoMap = info_map(), + Schemas = maps:get(action_type_to_schema_module, InfoMap), + All = maps:to_list(Schemas), + [{Type, SchemaMod} || {Type, SchemaMod} <- All, is_source(Type)]. connector_action_config_to_bridge_v1_config(ActionOrBridgeType, ConnectorConfig, ActionConfig) -> Module = get_action_info_module(ActionOrBridgeType), @@ -293,7 +329,9 @@ initial_info_map() -> action_type_to_bridge_v1_type => #{}, action_type_to_connector_type => #{}, action_type_to_schema_module => #{}, - action_type_to_info_module => #{} + action_type_to_info_module => #{}, + is_source => #{}, + is_action => #{} }. get_info_map(Module) -> @@ -312,6 +350,20 @@ get_info_map(Module) -> false -> {ActionType, [ActionType]} end, + IsIngress = + case erlang:function_exported(Module, is_source, 0) of + true -> + Module:is_source(); + false -> + false + end, + IsEgress = + case erlang:function_exported(Module, is_action, 0) of + true -> + Module:is_action(); + false -> + true + end, #{ action_type_names => lists:foldl( @@ -351,5 +403,11 @@ get_info_map(Module) -> end, #{ActionType => Module}, BridgeV1Types - ) + ), + is_source => #{ + ActionType => IsIngress + }, + is_action => #{ + ActionType => IsEgress + } }. diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d26a44a1d..c7d9a2d27 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -353,7 +353,13 @@ get_metrics(Type, Name) -> case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of true -> BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), - emqx_bridge_v2:get_metrics(BridgeV2Type, Name); + try + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one(Type, Name), + emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) + catch + error:Reason -> + {error, Reason} + end; false -> {error, not_bridge_v1_compatible} end; diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index e1cd03ac2..3168ae590 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -548,9 +548,17 @@ schema("/bridges_probe") -> Id, case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(BridgeV2Type, BridgeName), - ?NO_CONTENT; + try + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( + BridgeType, BridgeName + ), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), + ok = emqx_bridge_v2:reset_metrics(ConfRootKey, BridgeV2Type, BridgeName), + ?NO_CONTENT + catch + error:Reason -> + ?BAD_REQUEST(Reason) + end; false -> ok = emqx_bridge_resource:reset_metrics( emqx_bridge_resource:resource_id(BridgeType, BridgeName) diff --git a/apps/emqx_bridge/src/emqx_bridge_lib.erl b/apps/emqx_bridge/src/emqx_bridge_lib.erl index 9386a38d3..7f74dfb2d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_lib.erl +++ b/apps/emqx_bridge/src/emqx_bridge_lib.erl @@ -82,7 +82,8 @@ external_ids(Type, Name) -> get_conf(BridgeType, BridgeName) -> case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> - emqx_conf:get_raw([actions, BridgeType, BridgeName]); + ConfRootName = emqx_bridge_v2:get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_conf:get_raw([ConfRootName, BridgeType, BridgeName]); false -> undefined end. diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 723808919..66d4dc674 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -26,7 +26,8 @@ %% Note: this is strange right now, because it lives in `emqx_bridge_v2', but it shall be %% refactored into a new module/application with appropriate name. --define(ROOT_KEY, actions). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). %% Loading and unloading config when EMQX starts and stops -export([ @@ -38,7 +39,9 @@ -export([ list/0, + list/1, lookup/2, + lookup/3, create/3, %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies @@ -53,13 +56,16 @@ -export([ disable_enable/3, + disable_enable/4, health_check/2, send_message/4, query/4, start/2, reset_metrics/2, + reset_metrics/3, create_dry_run/2, - get_metrics/2 + get_metrics/2, + get_metrics/3 ]). %% On message publish hook (for local_topics) @@ -122,7 +128,8 @@ bridge_v1_stop/2, bridge_v1_start/2, %% For test cases only - bridge_v1_remove/2 + bridge_v1_remove/2, + get_conf_root_key_if_only_one/2 ]). %%==================================================================== @@ -151,19 +158,22 @@ %%==================================================================== load() -> - load_bridges(), + load_bridges(?ROOT_KEY_ACTIONS), + load_bridges(?ROOT_KEY_SOURCES), load_message_publish_hook(), ok = emqx_config_handler:add_handler(config_key_path_leaf(), emqx_bridge_v2), ok = emqx_config_handler:add_handler(config_key_path(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_leaf_sources(), emqx_bridge_v2), + ok = emqx_config_handler:add_handler(config_key_path_sources(), emqx_bridge_v2), ok. -load_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +load_bridges(RootName) -> + Bridges = emqx:get_config([RootName], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - install_bridge_v2(Type, Name, BridgeConf) + install_bridge_v2(RootName, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -172,19 +182,20 @@ load_bridges() -> ). unload() -> - unload_bridges(), + unload_bridges(?ROOT_KEY_ACTIONS), + unload_bridges(?ROOT_KEY_SOURCES), unload_message_publish_hook(), emqx_conf:remove_handler(config_key_path()), emqx_conf:remove_handler(config_key_path_leaf()), ok. -unload_bridges() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), +unload_bridges(ConfRooKey) -> + Bridges = emqx:get_config([ConfRooKey], #{}), lists:foreach( fun({Type, Bridge}) -> lists:foreach( fun({Name, BridgeConf}) -> - uninstall_bridge_v2(Type, Name, BridgeConf) + uninstall_bridge_v2(ConfRooKey, Type, Name, BridgeConf) end, maps:to_list(Bridge) ) @@ -198,7 +209,12 @@ unload_bridges() -> -spec lookup(bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(Type, Name) -> - case emqx:get_raw_config([?ROOT_KEY, Type, Name], not_found) of + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +-spec lookup(sources | actions, bridge_v2_type(), bridge_v2_name()) -> + {ok, bridge_v2_info()} | {error, not_found}. +lookup(ConfRootName, Type, Name) -> + case emqx:get_raw_config([ConfRootName, Type, Name], not_found) of not_found -> {error, not_found}; #{<<"connector">> := BridgeConnector} = RawConf -> @@ -218,7 +234,7 @@ lookup(Type, Name) -> %% Find the Bridge V2 status from the ConnectorData ConnectorStatus = maps:get(status, ConnectorData, undefined), Channels = maps:get(added_channels, ConnectorData, #{}), - BridgeV2Id = id(Type, Name, BridgeConnector), + BridgeV2Id = id_with_root_name(ConfRootName, Type, Name, BridgeConnector), ChannelStatus = maps:get(BridgeV2Id, Channels, undefined), {DisplayBridgeV2Status, ErrorMsg} = case {ChannelStatus, ConnectorStatus} of @@ -245,20 +261,30 @@ lookup(Type, Name) -> -spec list() -> [bridge_v2_info()] | {error, term()}. list() -> - list_with_lookup_fun(fun lookup/2). + list_with_lookup_fun(?ROOT_KEY_ACTIONS, fun lookup/2). + +list(ConfRootKey) -> + LookupFun = fun(Type, Name) -> + lookup(ConfRootKey, Type, Name) + end, + list_with_lookup_fun(ConfRootKey, LookupFun). -spec create(bridge_v2_type(), bridge_v2_name(), map()) -> {ok, emqx_config:update_result()} | {error, any()}. create(BridgeType, BridgeName, RawConf) -> + create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, RawConf). + +create(ConfRootKey, BridgeType, BridgeName, RawConf) -> ?SLOG(debug, #{ bridge_action => create, bridge_version => 2, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_utils:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf), + root_key_path => ConfRootKey }), emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], RawConf, #{override_to => cluster} ). @@ -267,6 +293,9 @@ create(BridgeType, BridgeName, RawConf) -> remove(BridgeType, BridgeName) -> %% NOTE: This function can cause broken references from rules but it is only %% called directly from test cases. + remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +remove(ConfRootKey, BridgeType, BridgeName) -> ?SLOG(debug, #{ bridge_action => remove, bridge_version => 2, @@ -275,7 +304,7 @@ remove(BridgeType, BridgeName) -> }), case emqx_conf:remove( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], #{override_to => cluster} ) of @@ -307,7 +336,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> %% Helpers for CRUD API %%-------------------------------------------------------------------- -list_with_lookup_fun(LookupFun) -> +list_with_lookup_fun(ConfRootName, LookupFun) -> maps:fold( fun(Type, NameAndConf, Bridges) -> maps:fold( @@ -330,21 +359,24 @@ list_with_lookup_fun(LookupFun) -> ) end, [], - emqx:get_raw_config([?ROOT_KEY], #{}) + emqx:get_raw_config([ConfRootName], #{}) ). install_bridge_v2( + _RootName, _BridgeType, _BridgeName, #{enable := false} ) -> ok; install_bridge_v2( + RootName, BridgeV2Type, BridgeName, Config ) -> install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, combine_connector_and_bridge_v2_config( @@ -355,6 +387,7 @@ install_bridge_v2( ). install_bridge_v2_helper( + _RootName, _BridgeV2Type, _BridgeName, {error, Reason} = Error @@ -362,11 +395,12 @@ install_bridge_v2_helper( ?SLOG(warning, Reason), Error; install_bridge_v2_helper( + RootName, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(RootName, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), %% Create metrics for Bridge V2 ok = emqx_resource:create_metrics(BridgeV2Id), @@ -388,18 +422,45 @@ install_bridge_v2_helper( ConnectorId = emqx_connector_resource:resource_id( connector_type(BridgeV2Type), ConnectorName ), - ConfigWithTypeAndName = Config#{ - bridge_type => bin(BridgeV2Type), - bridge_name => bin(BridgeName) - }, emqx_resource_manager:add_channel( ConnectorId, BridgeV2Id, - ConfigWithTypeAndName + augment_channel_config( + RootName, + BridgeV2Type, + BridgeName, + Config + ) ), ok. +augment_channel_config( + ConfigRoot, + BridgeV2Type, + BridgeName, + Config +) -> + AugmentedConf = Config#{ + config_root => ConfigRoot, + bridge_type => bin(BridgeV2Type), + bridge_name => bin(BridgeName) + }, + case emqx_action_info:is_source(BridgeV2Type) of + true -> + BId = emqx_bridge_resource:bridge_id(BridgeV2Type, BridgeName), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + SourceHookpoint = source_hookpoint(BId), + HookPoints = [BridgeHookpoint, SourceHookpoint], + AugmentedConf#{hookpoints => HookPoints}; + false -> + AugmentedConf + end. + +source_hookpoint(BridgeId) -> + <<"$sources/", (bin(BridgeId))/binary>>. + uninstall_bridge_v2( + _ConfRootKey, _BridgeType, _BridgeName, #{enable := false} @@ -407,11 +468,12 @@ uninstall_bridge_v2( %% Already not installed ok; uninstall_bridge_v2( + ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName} = Config ) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), CreationOpts = emqx_resource:fetch_creation_opts(Config), ok = emqx_resource_buffer_worker_sup:stop_workers(BridgeV2Id, CreationOpts), ok = emqx_resource:clear_metrics(BridgeV2Id), @@ -460,8 +522,11 @@ combine_connector_and_bridge_v2_config( -spec disable_enable(disable | enable, bridge_v2_type(), bridge_v2_name()) -> {ok, any()} | {error, any()}. disable_enable(Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> + disable_enable(?ROOT_KEY_ACTIONS, Action, BridgeType, BridgeName). + +disable_enable(ConfRootKey, Action, BridgeType, BridgeName) when ?ENABLE_OR_DISABLE(Action) -> emqx_conf:update( - config_key_path() ++ [BridgeType, BridgeName], + [ConfRootKey, BridgeType, BridgeName], Action, #{override_to => cluster} ). @@ -477,10 +542,12 @@ start(BridgeV2Type, Name) -> ConnectorOpFun = fun(ConnectorType, ConnectorName) -> emqx_connector_resource:start(ConnectorType, ConnectorName) end, - connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, true). + ConfRootKey = ?ROOT_KEY_ACTIONS, + connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, true). -connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> +connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, lookup_conf(BridgeV2Type, Name), @@ -489,14 +556,16 @@ connector_operation_helper(BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> ). connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, - {error, bridge_not_found} = Error, + {error, _} = Error, _ConnectorOpFun, _DoHealthCheck ) -> Error; connector_operation_helper_with_conf( + _ConfRootKey, _BridgeV2Type, _Name, #{enable := false}, @@ -505,6 +574,7 @@ connector_operation_helper_with_conf( ) -> ok; connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, #{connector := ConnectorName}, @@ -519,7 +589,7 @@ connector_operation_helper_with_conf( {true, {error, Reason}} -> {error, Reason}; {true, ok} -> - case health_check(BridgeV2Type, Name) of + case health_check(ConfRootKey, BridgeV2Type, Name) of #{status := connected} -> ok; {error, Reason} -> @@ -536,14 +606,17 @@ connector_operation_helper_with_conf( -spec reset_metrics(bridge_v2_type(), bridge_v2_name()) -> ok | {error, not_found}. reset_metrics(Type, Name) -> - reset_metrics_helper(Type, Name, lookup_conf(Type, Name)). + reset_metrics(?ROOT_KEY_ACTIONS, Type, Name). -reset_metrics_helper(_Type, _Name, #{enable := false}) -> +reset_metrics(ConfRootKey, Type, Name) -> + reset_metrics_helper(ConfRootKey, Type, Name, lookup_conf(ConfRootKey, Type, Name)). + +reset_metrics_helper(_ConfRootKey, _Type, _Name, #{enable := false}) -> ok; -reset_metrics_helper(BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> - BridgeV2Id = id(BridgeV2Type, BridgeName, ConnectorName), +reset_metrics_helper(ConfRootKey, BridgeV2Type, BridgeName, #{connector := ConnectorName}) -> + BridgeV2Id = id_with_root_name(ConfRootKey, BridgeV2Type, BridgeName, ConnectorName), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, BridgeV2Id); -reset_metrics_helper(_, _, _) -> +reset_metrics_helper(_, _, _, _) -> {error, not_found}. get_query_mode(BridgeV2Type, Config) -> @@ -599,7 +672,10 @@ send_message(BridgeType, BridgeName, Message, QueryOpts0) -> -spec health_check(BridgeType :: term(), BridgeName :: term()) -> #{status := emqx_resource:resource_status(), error := term()} | {error, Reason :: term()}. health_check(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of + health_check(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +health_check(ConfRootKey, BridgeType, BridgeName) -> + case lookup_conf(ConfRootKey, BridgeType, BridgeName) of #{ enable := true, connector := ConnectorName @@ -608,7 +684,7 @@ health_check(BridgeType, BridgeName) -> connector_type(BridgeType), ConnectorName ), emqx_resource_manager:channel_health_check( - ConnectorId, id(BridgeType, BridgeName, ConnectorName) + ConnectorId, id_with_root_name(ConfRootKey, BridgeType, BridgeName, ConnectorName) ); #{enable := false} -> {error, bridge_stopped}; @@ -652,13 +728,13 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> {_, ConnectorName} = emqx_connector_resource:parse_connector_id(ConnectorId), ChannelTestId = id(BridgeType, BridgeName, ConnectorName), Conf = emqx_utils_maps:unsafe_atom_key_map(BridgeV2RawConf), - ConfWithTypeAndName = Conf#{ - bridge_type => bin(BridgeType), - bridge_name => bin(BridgeName) - }, - case - emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, ConfWithTypeAndName) - of + AugmentedConf = augment_channel_config( + ?ROOT_KEY_ACTIONS, + BridgeType, + BridgeName, + Conf + ), + case emqx_resource_manager:add_channel(ConnectorId, ChannelTestId, AugmentedConf) of {error, Reason} -> {error, Reason}; ok -> @@ -677,7 +753,10 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> -spec get_metrics(bridge_v2_type(), bridge_v2_name()) -> emqx_metrics_worker:metrics(). get_metrics(Type, Name) -> - emqx_resource:get_metrics(id(Type, Name)). + get_metrics(?ROOT_KEY_ACTIONS, Type, Name). + +get_metrics(ConfRootKey, Type, Name) -> + emqx_resource:get_metrics(id_with_root_name(ConfRootKey, Type, Name)). %%==================================================================== %% On message publish hook (for local topics) @@ -690,7 +769,7 @@ reload_message_publish_hook(Bridges) -> ok = load_message_publish_hook(Bridges). load_message_publish_hook() -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), load_message_publish_hook(Bridges). load_message_publish_hook(Bridges) -> @@ -754,7 +833,7 @@ send_to_matched_egress_bridges(Topic, Msg) -> ). get_matched_egress_bridges(Topic) -> - Bridges = emqx:get_config([?ROOT_KEY], #{}), + Bridges = emqx:get_config([?ROOT_KEY_ACTIONS], #{}), maps:fold( fun(BType, Conf, Acc0) -> maps:fold( @@ -800,16 +879,21 @@ parse_id(Id) -> end. get_channels_for_connector(ConnectorId) -> + Actions = get_channels_for_connector(?ROOT_KEY_ACTIONS, ConnectorId), + Sources = get_channels_for_connector(?ROOT_KEY_SOURCES, ConnectorId), + Actions ++ Sources. + +get_channels_for_connector(SourcesOrActions, ConnectorId) -> try emqx_connector_resource:parse_connector_id(ConnectorId) of {ConnectorType, ConnectorName} -> - RootConf = maps:keys(emqx:get_config([?ROOT_KEY], #{})), + RootConf = maps:keys(emqx:get_config([SourcesOrActions], #{})), RelevantBridgeV2Types = [ Type || Type <- RootConf, connector_type(Type) =:= ConnectorType ], lists:flatten([ - get_channels_for_connector(ConnectorName, BridgeV2Type) + get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) || BridgeV2Type <- RelevantBridgeV2Types ]) catch @@ -819,33 +903,55 @@ get_channels_for_connector(ConnectorId) -> [] end. -get_channels_for_connector(ConnectorName, BridgeV2Type) -> - BridgeV2s = emqx:get_config([?ROOT_KEY, BridgeV2Type], #{}), +get_channels_for_connector(SourcesOrActions, ConnectorName, BridgeV2Type) -> + BridgeV2s = emqx:get_config([SourcesOrActions, BridgeV2Type], #{}), [ - {id(BridgeV2Type, Name, ConnectorName), Conf#{ - bridge_name => bin(Name), - bridge_type => bin(BridgeV2Type) - }} + { + id_with_root_name(SourcesOrActions, BridgeV2Type, Name, ConnectorName), + augment_channel_config(SourcesOrActions, BridgeV2Type, Name, Conf) + } || {Name, Conf} <- maps:to_list(BridgeV2s), bin(ConnectorName) =:= maps:get(connector, Conf, no_name) ]. %%==================================================================== -%% Exported for tests +%% ID related functions %%==================================================================== id(BridgeType, BridgeName) -> - case lookup_conf(BridgeType, BridgeName) of - #{connector := ConnectorName} -> - id(BridgeType, BridgeName, ConnectorName); - {error, Reason} -> - throw(Reason) - end. + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). id(BridgeType, BridgeName, ConnectorName) -> + id_with_root_name(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, ConnectorName). + +id_with_root_name(RootName, BridgeType, BridgeName) -> + case lookup_conf(RootName, BridgeType, BridgeName) of + #{connector := ConnectorName} -> + id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName); + {error, Reason} -> + throw( + {action_source_not_found, #{ + reason => Reason, + root_name => RootName, + type => BridgeType, + name => BridgeName + }} + ) + end. + +id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName) -> ConnectorType = bin(connector_type(BridgeType)), - <<"action:", (bin(BridgeType))/binary, ":", (bin(BridgeName))/binary, ":connector:", - (bin(ConnectorType))/binary, ":", (bin(ConnectorName))/binary>>. + << + (bin(RootName))/binary, + ":", + (bin(BridgeType))/binary, + ":", + (bin(BridgeName))/binary, + ":connector:", + (bin(ConnectorType))/binary, + ":", + (bin(ConnectorName))/binary + >>. connector_type(Type) -> %% remote call so it can be mocked @@ -860,76 +966,65 @@ bridge_v2_type_to_connector_type(Type) -> import_config(RawConf) -> %% actions structure - emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY, config_key_path()). + emqx_bridge:import_config(RawConf, <<"actions">>, ?ROOT_KEY_ACTIONS, config_key_path()). %%==================================================================== %% Config Update Handler API %%==================================================================== -config_key_path() -> [?ROOT_KEY]. +config_key_path() -> + [?ROOT_KEY_ACTIONS]. -config_key_path_leaf() -> [?ROOT_KEY, '?', '?']. +config_key_path_leaf() -> + [?ROOT_KEY_ACTIONS, '?', '?']. + +config_key_path_sources() -> + [?ROOT_KEY_SOURCES]. + +config_key_path_leaf_sources() -> + [?ROOT_KEY_SOURCES, '?', '?']. %% enable or disable action -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, undefined) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, undefined) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {error, bridge_not_found}; -pre_config_update([?ROOT_KEY, _Type, _Name], Oper, OldAction) when ?ENABLE_OR_DISABLE(Oper) -> +pre_config_update([ConfRootKey, _Type, _Name], Oper, OldAction) when + ?ENABLE_OR_DISABLE(Oper) andalso + (ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES) +-> {ok, OldAction#{<<"enable">> => operation_to_enable(Oper)}}; %% Updates a single action from a specific HTTP API. %% If the connector is not found, the update operation fails. -pre_config_update([?ROOT_KEY, Type, Name], Conf = #{}, _OldConf) -> - action_convert_from_connector(Type, Name, Conf); +pre_config_update([ConfRootKey, Type, Name], Conf = #{}, _OldConf) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + convert_from_connector(ConfRootKey, Type, Name, Conf); %% Batch updates actions when importing a configuration or executing a CLI command. %% Update succeeded even if the connector is not found, alarm in post_config_update -pre_config_update([?ROOT_KEY], Conf = #{}, _OldConfs) -> - {ok, actions_convert_from_connectors(Conf)}. +pre_config_update([ConfRootKey], Conf = #{}, _OldConfs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS orelse ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + {ok, convert_from_connectors(ConfRootKey, Conf)}. -%% Don't crash event the bridge is not found -post_config_update([?ROOT_KEY, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> - AllBridges = emqx:get_config([?ROOT_KEY]), - case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of - undefined -> - ok; - Action -> - ok = uninstall_bridge_v2(Type, Name, Action), - Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), - reload_message_publish_hook(Bridges) - end, - ?tp(bridge_post_config_update_done, #{}), - ok; -%% Create a single bridge failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) -> - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; -%% update bridges failed if the connector is not found(already check in pre_config_update) -post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) -> - ok = uninstall_bridge_v2(BridgeType, BridgeName, OldConf), - ok = install_bridge_v2(BridgeType, BridgeName, NewConf), - Bridges = emqx_utils_maps:deep_put( - [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf - ), - reload_message_publish_hook(Bridges), - ?tp(bridge_post_config_update_done, #{}), - ok; %% This top level handler will be triggered when the actions path is updated %% with calls to emqx_conf:update([actions], BridgesConf, #{}). -%% such as import_config/1 -%% Notice ** do succeeded even if the connector is not found ** -%% Install a non-exist connector will alarm & log(warn) in install_bridge_v2. -post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> +post_config_update([ConfRootKey], _Req, NewConf, OldConf, _AppEnv) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), %% The config update will be failed if any task in `perform_bridge_changes` failed. - RemoveFun = fun uninstall_bridge_v2/3, - CreateFun = fun install_bridge_v2/3, + RemoveFun = fun(Type, Name, Conf) -> + uninstall_bridge_v2(ConfRootKey, Type, Name, Conf) + end, + CreateFun = fun(Type, Name, Conf) -> + install_bridge_v2(ConfRootKey, Type, Name, Conf) + end, UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) -> - uninstall_bridge_v2(Type, Name, OldBridgeConf), - install_bridge_v2(Type, Name, Conf) + uninstall_bridge_v2(ConfRootKey, Type, Name, OldBridgeConf), + install_bridge_v2(ConfRootKey, Type, Name, Conf) end, Result = perform_bridge_changes([ #{action => RemoveFun, data => Removed}, @@ -942,7 +1037,45 @@ post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> ]), reload_message_publish_hook(NewConf), ?tp(bridge_post_config_update_done, #{}), - Result. + Result; +%% Don't crash even when the bridge is not found +post_config_update([ConfRootKey, Type, Name], '$remove', _, _OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + AllBridges = emqx:get_config([ConfRootKey]), + case emqx_utils_maps:deep_get([Type, Name], AllBridges, undefined) of + undefined -> + ok; + Action -> + ok = uninstall_bridge_v2(ConfRootKey, Type, Name, Action), + Bridges = emqx_utils_maps:deep_remove([Type, Name], AllBridges), + reload_message_publish_hook(Bridges) + end, + ?tp(bridge_post_config_update_done, #{}), + ok; +%% Create a single bridge fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok; +%% update bridges fails if the connector is not found (already checked in pre_config_update) +post_config_update([ConfRootKey, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) when + ConfRootKey =:= ?ROOT_KEY_ACTIONS; ConfRootKey =:= ?ROOT_KEY_SOURCES +-> + ok = uninstall_bridge_v2(ConfRootKey, BridgeType, BridgeName, OldConf), + ok = install_bridge_v2(ConfRootKey, BridgeType, BridgeName, NewConf), + Bridges = emqx_utils_maps:deep_put( + [BridgeType, BridgeName], emqx:get_config([ConfRootKey]), NewConf + ), + reload_message_publish_hook(Bridges), + ?tp(bridge_post_config_update_done, #{}), + ok. diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( @@ -1051,12 +1184,33 @@ is_bridge_v2_type(Type) -> emqx_action_info:is_action_type(Type). bridge_v1_list_and_transform() -> - Bridges = list_with_lookup_fun(fun bridge_v1_lookup_and_transform/2), - [B || B <- Bridges, B =/= not_bridge_v1_compatible_error()]. + BridgesFromActions0 = list_with_lookup_fun( + ?ROOT_KEY_ACTIONS, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromActions1 = [ + B + || B <- BridgesFromActions0, + B =/= not_bridge_v1_compatible_error() + ], + FromActionsNames = maps:from_keys([Name || #{name := Name} <- BridgesFromActions1], true), + BridgesFromSources0 = list_with_lookup_fun( + ?ROOT_KEY_SOURCES, + fun bridge_v1_lookup_and_transform/2 + ), + BridgesFromSources1 = [ + B + || #{name := SourceBridgeName} = B <- BridgesFromSources0, + B =/= not_bridge_v1_compatible_error(), + %% Action is only shown in case of name conflict + not maps:is_key(SourceBridgeName, FromActionsNames) + ], + BridgesFromActions1 ++ BridgesFromSources1. bridge_v1_lookup_and_transform(ActionType, Name) -> - case lookup(ActionType, Name) of - {ok, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> + case lookup_actions_or_sources(ActionType, Name) of + {ok, ConfRootName, + #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), HasBridgeV1Equivalent = has_bridge_v1_equivalent(ActionType), case HasBridgeV1Equivalent andalso ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of @@ -1065,6 +1219,7 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( + ConfRootName, BridgeV1Type, Name, ActionType, @@ -1082,6 +1237,19 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> Error end. +lookup_actions_or_sources(ActionType, Name) -> + case lookup(?ROOT_KEY_ACTIONS, ActionType, Name) of + {error, not_found} -> + case lookup(?ROOT_KEY_SOURCES, ActionType, Name) of + {ok, SourceInfo} -> + {ok, ?ROOT_KEY_SOURCES, SourceInfo}; + Error -> + Error + end; + {ok, ActionInfo} -> + {ok, ?ROOT_KEY_ACTIONS, ActionInfo} + end. + not_bridge_v1_compatible_error() -> {error, not_bridge_v1_compatible}. @@ -1094,18 +1262,18 @@ has_bridge_v1_equivalent(ActionType) -> connector_raw_config(Connector, ConnectorType) -> get_raw_with_defaults(Connector, ConnectorType, <<"connectors">>, emqx_connector_schema). -action_raw_config(Action, ActionType) -> - get_raw_with_defaults(Action, ActionType, <<"actions">>, emqx_bridge_v2_schema). +action_raw_config(ConfRootName, Action, ActionType) -> + get_raw_with_defaults(Action, ActionType, bin(ConfRootName), emqx_bridge_v2_schema). get_raw_with_defaults(Config, Type, TopLevelConf, SchemaModule) -> RawConfig = maps:get(raw_config, Config), fill_defaults(Type, RawConfig, TopLevelConf, SchemaModule). bridge_v1_lookup_and_transform_helper( - BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector + ConfRootName, BridgeV1Type, BridgeName, ActionType, Action, ConnectorType, Connector ) -> ConnectorRawConfig = connector_raw_config(Connector, ConnectorType), - ActionRawConfig = action_raw_config(Action, ActionType), + ActionRawConfig = action_raw_config(ConfRootName, Action, ActionType), BridgeV1Config = emqx_action_info:connector_action_config_to_bridge_v1_config( BridgeV1Type, ConnectorRawConfig, ActionRawConfig ), @@ -1136,7 +1304,52 @@ bridge_v1_lookup_and_transform_helper( end. lookup_conf(Type, Name) -> - case emqx:get_config([?ROOT_KEY, Type, Name], not_found) of + lookup_conf(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_conf_if_one_of_sources_actions(Type, Name) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, Type, Name), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, Type, Name), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + {error, bridge_not_found}; + {{error, bridge_not_found}, Conf} -> + Conf; + {Conf, {error, bridge_not_found}} -> + Conf; + {_Conf1, _Conf2} -> + {error, name_conflict_sources_actions} + end. + +is_only_source(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + false; + {{error, bridge_not_found}, _Conf} -> + true; + {_Conf, {error, bridge_not_found}} -> + false; + {_Conf1, _Conf2} -> + false + end. + +get_conf_root_key_if_only_one(BridgeType, BridgeName) -> + LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, BridgeType, BridgeName), + LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), + case {LookUpConfActions, LookUpConfSources} of + {{error, bridge_not_found}, {error, bridge_not_found}} -> + error({action_or_soruces_not_found, BridgeType, BridgeName}); + {{error, bridge_not_found}, _Conf} -> + ?ROOT_KEY_SOURCES; + {_Conf, {error, bridge_not_found}} -> + ?ROOT_KEY_ACTIONS; + {_Conf1, _Conf2} -> + error({name_clash_action_soruces, BridgeType, BridgeName}) + end. + +lookup_conf(RootName, Type, Name) -> + case emqx:get_config([RootName, Type, Name], not_found) of not_found -> {error, bridge_not_found}; Config -> @@ -1146,7 +1359,7 @@ lookup_conf(Type, Name) -> bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), %% Check if the bridge v2 exists - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, @@ -1158,8 +1371,9 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> true -> %% Using remove + create as update, hence do not delete deps. RemoveDeps = [], + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, BridgeName), PreviousRawConf = emqx:get_raw_config( - [?ROOT_KEY, BridgeV2Type, BridgeName], undefined + [ConfRootKey, BridgeV2Type, BridgeName], undefined ), %% To avoid losing configurations. We have to make sure that no crash occurs %% during deletion and creation of configurations. @@ -1185,17 +1399,18 @@ split_bridge_v1_config_and_create_helper( connector_conf := NewConnectorRawConf, bridge_v2_type := BridgeType, bridge_v2_name := BridgeName, - bridge_v2_conf := NewBridgeV2RawConf + bridge_v2_conf := NewBridgeV2RawConf, + conf_root_key := ConfRootName } = split_and_validate_bridge_v1_config( BridgeV1Type, BridgeName, RawConf, PreviousRawConf ), - _ = PreCreateFun(), do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1210,6 +1425,7 @@ split_bridge_v1_config_and_create_helper( end. do_connector_and_bridge_create( + ConfRootName, ConnectorType, NewConnectorName, NewConnectorRawConf, @@ -1220,7 +1436,7 @@ do_connector_and_bridge_create( ) -> case emqx_connector:create(ConnectorType, NewConnectorName, NewConnectorRawConf) of {ok, _} -> - case create(BridgeType, BridgeName, NewBridgeV2RawConf) of + case create(ConfRootName, BridgeType, BridgeName, NewBridgeV2RawConf) of {ok, _} = Result -> Result; {error, Reason1} -> @@ -1257,10 +1473,15 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR } } }, + ConfRootKeyPrevRawConf = + case PreviousRawConf =/= undefined of + true -> get_conf_root_key_if_only_one(BridgeV2Type, BridgeName); + false -> not_used + end, FakeGlobalConfig = emqx_utils_maps:put_if( FakeGlobalConfig0, - bin(?ROOT_KEY), + bin(ConfRootKeyPrevRawConf), #{bin(BridgeV2Type) => #{bin(BridgeName) => PreviousRawConf}}, PreviousRawConf =/= undefined ), @@ -1269,10 +1490,11 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR Output = emqx_connector_schema:transform_bridges_v1_to_connectors_and_bridges_v2( FakeGlobalConfig ), + ConfRootKey = get_conf_root_key(Output), NewBridgeV2RawConf = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName) ], @@ -1280,7 +1502,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR ), ConnectorName = emqx_utils_maps:deep_get( [ - bin(?ROOT_KEY), + ConfRootKey, bin(BridgeV2Type), bin(BridgeName), <<"connector">> @@ -1303,7 +1525,7 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR bin(ConnectorName) => NewConnectorRawConf } }, - <<"actions">> => #{ + ConfRootKey => #{ bin(BridgeV2Type) => #{ bin(BridgeName) => NewBridgeV2RawConf } @@ -1323,7 +1545,8 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR connector_conf => NewConnectorRawConf, bridge_v2_type => BridgeV2Type, bridge_v2_name => BridgeName, - bridge_v2_conf => NewBridgeV2RawConf + bridge_v2_conf => NewBridgeV2RawConf, + conf_root_key => ConfRootKey } catch %% validation errors @@ -1331,6 +1554,13 @@ split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousR {error, Reason1} end. +get_conf_root_key(#{<<"actions">> := _}) -> + <<"actions">>; +get_conf_root_key(#{<<"sources">> := _}) -> + <<"sources">>; +get_conf_root_key(_NoMatch) -> + error({incompatible_bridge_v1, no_action_or_source}). + bridge_v1_create_dry_run(BridgeType, RawConfig0) -> RawConf = maps:without([<<"name">>], RawConfig0), TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), @@ -1356,7 +1586,7 @@ bridge_v1_remove(BridgeV1Type, BridgeName) -> bridge_v1_remove( ActionType, BridgeName, - lookup_conf(ActionType, BridgeName) + lookup_conf_if_one_of_sources_actions(ActionType, BridgeName) ). bridge_v1_remove( @@ -1364,7 +1594,8 @@ bridge_v1_remove( Name, #{connector := ConnectorName} ) -> - case remove(ActionType, Name) of + ConfRootKey = get_conf_root_key_if_only_one(ActionType, Name), + case remove(ConfRootKey, ActionType, Name) of ok -> ConnectorType = connector_type(ActionType), emqx_connector:remove(ConnectorType, ConnectorName); @@ -1384,7 +1615,7 @@ bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) -> BridgeV2Type, BridgeName, RemoveDeps, - lookup_conf(BridgeV2Type, BridgeName) + lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) ). %% Bridge v1 delegated-removal in 3 steps: @@ -1398,9 +1629,10 @@ bridge_v1_check_deps_and_remove( #{connector := ConnectorName} ) -> RemoveConnector = lists:member(connector, RemoveDeps), - case emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of + case maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) of ok -> - case remove(BridgeType, BridgeName) of + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + case remove(ConfRootKey, BridgeType, BridgeName) of ok when RemoveConnector -> maybe_delete_channels(BridgeType, BridgeName, ConnectorName); ok -> @@ -1415,6 +1647,14 @@ bridge_v1_check_deps_and_remove(_BridgeType, _BridgeName, _RemoveDeps, Error) -> %% TODO: the connector is gone, for whatever reason, maybe call remove/2 anyway? Error. +maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) -> + case is_only_source(BridgeType, BridgeName) of + true -> + ok; + false -> + emqx_bridge_lib:maybe_withdraw_rule_action(BridgeType, BridgeName, RemoveDeps) + end. + maybe_delete_channels(BridgeType, BridgeName, ConnectorName) -> case connector_has_channels(BridgeType, ConnectorName) of true -> @@ -1467,23 +1707,25 @@ bridge_v1_enable_disable(Action, BridgeType, BridgeName) -> Action, BridgeType, BridgeName, - lookup_conf(BridgeType, BridgeName) + lookup_conf_if_one_of_sources_actions(BridgeType, BridgeName) ); false -> {error, not_bridge_v1_compatible} end. -bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, bridge_not_found}) -> - {error, bridge_not_found}; +bridge_v1_enable_disable_helper(_Op, _BridgeType, _BridgeName, {error, Reason}) -> + {error, Reason}; bridge_v1_enable_disable_helper(enable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), {ok, _} = emqx_connector:disable_enable(enable, ConnectorType, ConnectorName), - emqx_bridge_v2:disable_enable(enable, BridgeV2Type, BridgeName); + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + emqx_bridge_v2:disable_enable(ConfRootKey, enable, BridgeV2Type, BridgeName); bridge_v1_enable_disable_helper(disable, BridgeType, BridgeName, #{connector := ConnectorName}) -> BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), ConnectorType = connector_type(BridgeV2Type), - {ok, _} = emqx_bridge_v2:disable_enable(disable, BridgeV2Type, BridgeName), + ConfRootKey = get_conf_root_key_if_only_one(BridgeType, BridgeName), + {ok, _} = emqx_bridge_v2:disable_enable(ConfRootKey, disable, BridgeV2Type, BridgeName), emqx_connector:disable_enable(disable, ConnectorType, ConnectorName). bridge_v1_restart(BridgeV1Type, Name) -> @@ -1508,10 +1750,12 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), case emqx_bridge_v2:bridge_v1_is_valid(BridgeV1Type, Name) of true -> + ConfRootKey = get_conf_root_key_if_only_one(BridgeV2Type, Name), connector_operation_helper_with_conf( + ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf_if_one_of_sources_actions(BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ); @@ -1559,12 +1803,12 @@ referenced_connectors_exist(BridgeType, ConnectorNameBin, BridgeName) -> ok end. -actions_convert_from_connectors(Conf) -> +convert_from_connectors(ConfRootKey, Conf) -> maps:map( fun(ActionType, Actions) -> maps:map( fun(ActionName, Action) -> - case action_convert_from_connector(ActionType, ActionName, Action) of + case convert_from_connector(ConfRootKey, ActionType, ActionName, Action) of {ok, NewAction} -> NewAction; {error, _} -> Action end @@ -1575,7 +1819,7 @@ actions_convert_from_connectors(Conf) -> Conf ). -action_convert_from_connector(Type, Name, Action = #{<<"connector">> := ConnectorName}) -> +convert_from_connector(ConfRootKey, Type, Name, Action = #{<<"connector">> := ConnectorName}) -> case get_connector_info(ConnectorName, Type) of {ok, Connector} -> Action1 = emqx_action_info:action_convert_from_connector(Type, Connector, Action), @@ -1585,7 +1829,8 @@ action_convert_from_connector(Type, Name, Action = #{<<"connector">> := Connecto bridge_name => Name, reason => <<"connector_not_found_or_wrong_type">>, bridge_type => Type, - connector_name => ConnectorName + connector_name => ConnectorName, + conf_root_key => ConfRootKey }} end. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index e144f332d..28017f814 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -79,7 +79,7 @@ api_schema(Method) -> hoconsc:union(bridge_api_union(APISchemas)). registered_api_schemas(Method) -> - RegisteredSchemas = emqx_action_info:registered_schema_modules(), + RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_bridge_v2") || {BridgeV2Type, SchemaModule} <- RegisteredSchemas @@ -189,29 +189,43 @@ tags() -> -dialyzer({nowarn_function, roots/0}). roots() -> - case fields(actions) of - [] -> - [ - {actions, - ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} - ]; - _ -> - [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] - end. + ActionsRoot = + case fields(actions) of + [] -> + [ + {actions, + ?HOCON(hoconsc:map(name, typerefl:map()), #{importance => ?IMPORTANCE_LOW})} + ]; + _ -> + [{actions, ?HOCON(?R_REF(actions), #{importance => ?IMPORTANCE_LOW})}] + end, + SourcesRoot = + [{sources, ?HOCON(?R_REF(sources), #{importance => ?IMPORTANCE_LOW})}], + ActionsRoot ++ SourcesRoot. fields(actions) -> - registered_schema_fields(); + registered_schema_fields_actions(); +fields(sources) -> + registered_schema_fields_sources(); fields(resource_opts) -> resource_opts_fields(_Overrides = []). -registered_schema_fields() -> +registered_schema_fields_actions() -> [ Module:fields(action) - || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules() + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_actions() + ]. + +registered_schema_fields_sources() -> + [ + Module:fields(source) + || {_BridgeV2Type, Module} <- emqx_action_info:registered_schema_modules_sources() ]. desc(actions) -> ?DESC("desc_bridges_v2"); +desc(sources) -> + ?DESC("desc_sources"); desc(resource_opts) -> ?DESC(emqx_resource_schema, "resource_opts"); desc(_) -> @@ -264,7 +278,7 @@ examples(Method) -> ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), lists:foldl(MergeFun, Examples, ConnectorExamples) end, - SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules()], + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], lists:foldl(Fun, #{}, SchemaModules). top_level_common_action_keys() -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index cc2296d3c..1bee2c92e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -20,8 +20,13 @@ -include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). +-behaviour(ecpool_worker). + +%% ecpool +-export([connect/1]). -export([on_message_received/3]). +-export([handle_disconnect/1]). %% callbacks of behaviour emqx_resource -export([ @@ -30,11 +35,25 @@ on_stop/2, on_query/3, on_query_async/4, - on_get_status/2 + on_get_status/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channel_status/3, + on_get_channels/1 ]). -export([on_async_result/2]). +-type name() :: term(). + +-type option() :: + {name, name()} + | {ingress, map()} + %% see `emqtt:option()` + | {client_opts, map()}. + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -define(HEALTH_CHECK_TIMEOUT, 1000). -define(INGRESS, "I"). -define(EGRESS, "E"). @@ -42,142 +61,205 @@ %% =================================================================== %% When use this bridge as a data source, ?MODULE:on_message_received will be called %% if the bridge received msgs from the remote broker. -on_message_received(Msg, HookPoint, ResId) -> + +on_message_received(Msg, HookPoints, ResId) -> emqx_resource_metrics:received_inc(ResId), - emqx_hooks:run(HookPoint, [Msg]). + lists:foreach( + fun(HookPoint) -> + emqx_hooks:run(HookPoint, [Msg]) + end, + HookPoints + ), + ok. %% =================================================================== callback_mode() -> async_if_possible. -on_start(ResourceId, Conf) -> +on_start(ResourceId, #{server := Server} = Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => ResourceId, config => emqx_utils:redact(Conf) }), - case start_ingress(ResourceId, Conf) of + TopicToHandlerIndex = emqx_topic_index:new(), + StartConf = Conf#{topic_to_handler_index => TopicToHandlerIndex}, + case start_mqtt_clients(ResourceId, StartConf) of {ok, Result1} -> - case start_egress(ResourceId, Conf) of - {ok, Result2} -> - {ok, maps:merge(Result1, Result2)}; - {error, Reason} -> - _ = stop_ingress(Result1), - {error, Reason} - end; + {ok, Result1#{ + installed_channels => #{}, + clean_start => maps:get(clean_start, Conf), + topic_to_handler_index => TopicToHandlerIndex, + server => Server + }}; {error, Reason} -> {error, Reason} end. -start_ingress(ResourceId, Conf) -> - ClientOpts = mk_client_opts(ResourceId, ?INGRESS, Conf), - case mk_ingress_config(ResourceId, Conf) of - Ingress = #{} -> - start_ingress(ResourceId, Ingress, ClientOpts); - undefined -> - {ok, #{}} - end. - -start_ingress(ResourceId, Ingress, ClientOpts) -> - PoolName = <>, - PoolSize = choose_ingress_pool_size(ResourceId, Ingress), - Options = [ - {name, PoolName}, - {pool_size, PoolSize}, - {ingress, Ingress}, - {client_opts, ClientOpts} - ], - ok = emqx_resource:allocate_resource(ResourceId, ingress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_ingress, Options) of - ok -> - {ok, #{ingress_pool_name => PoolName}}; - {error, {start_pool_failed, _, Reason}} -> - {error, Reason} - end. - -choose_ingress_pool_size(<>, _) -> - 1; -choose_ingress_pool_size( - ResourceId, - #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} +on_add_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + clean_start := CleanStart + } = OldState, + ChannelId, + #{config_root := actions} = ChannelConfig ) -> - case emqx_topic:parse(RemoteTopic) of - {#share{} = _Filter, _SubOpts} -> - % NOTE: this is shared subscription, many workers may subscribe - PoolSize; - {_Filter, #{}} when PoolSize > 1 -> - % NOTE: this is regular subscription, only one worker should subscribe + %% Publisher channel + %% make a warning if clean_start is set to false + case CleanStart of + false -> + ?tp( + mqtt_clean_start_egress_action_warning, + #{ + channel_id => ChannelId, + resource_id => _InstId + } + ), ?SLOG(warning, #{ - msg => "mqtt_bridge_ingress_pool_size_ignored", - connector => ResourceId, - reason => - "Remote topic filter is not a shared subscription, " - "ingress pool will start with a single worker", - config_pool_size => PoolSize, - pool_size => 1 - }), - 1; - {_Filter, #{}} when PoolSize == 1 -> - 1 - end. + msg => "mqtt_publisher_clean_start_false", + reason => "clean_start is set to false when using MQTT publisher action, " ++ + "which may cause unexpected behavior. " ++ + "For example, if the client ID is already subscribed to topics, " ++ + "we might receive messages that are unhanded.", + channel => ChannelId, + config => emqx_utils:redact(ChannelConfig) + }); + true -> + ok + end, + ChannelState0 = maps:get(parameters, ChannelConfig), + ChannelState = emqx_bridge_mqtt_egress:config(ChannelState0), + NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}; +on_add_channel( + _ResourceId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex, + server := Server + } = OldState, + ChannelId, + #{hookpoints := HookPoints} = ChannelConfig +) -> + %% Add ingress channel + ChannelState0 = maps:get(parameters, ChannelConfig), + ChannelState1 = ChannelState0#{ + hookpoints => HookPoints, + server => Server, + config_root => sources + }, + ChannelState2 = mk_ingress_config(ChannelId, ChannelState1, TopicToHandlerIndex), + ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState2), + NewInstalledChannels = maps:put(ChannelId, ChannelState2, InstalledChannels), + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Conf) -> - % NOTE - % We are ignoring the user configuration here because there's currently no reliable way - % to ensure proper session recovery according to the MQTT spec. - ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, ?EGRESS, Conf)), - case mk_egress_config(Conf) of - Egress = #{} -> - start_egress(ResourceId, Egress, ClientOpts); - undefined -> - {ok, #{}} - end. +on_remove_channel( + _InstId, + #{ + installed_channels := InstalledChannels, + pool_name := PoolName, + topic_to_handler_index := TopicToHandlerIndex + } = OldState, + ChannelId +) -> + ChannelState = maps:get(ChannelId, InstalledChannels), + case ChannelState of + #{ + config_root := sources + } -> + emqx_bridge_mqtt_ingress:unsubscribe_channel( + PoolName, ChannelState, ChannelId, TopicToHandlerIndex + ), + ok; + _ -> + ok + end, + NewInstalledChannels = maps:remove(ChannelId, InstalledChannels), + %% Update state + NewState = OldState#{installed_channels => NewInstalledChannels}, + {ok, NewState}. -start_egress(ResourceId, Egress, ClientOpts) -> - PoolName = <>, - PoolSize = maps:get(pool_size, Egress), +on_get_channel_status( + _ResId, + ChannelId, + #{ + installed_channels := Channels + } = _State +) when is_map_key(ChannelId, Channels) -> + %% The channel should be ok as long as the MQTT client is ok + connected. + +on_get_channels(ResId) -> + emqx_bridge_v2:get_channels_for_connector(ResId). + +start_mqtt_clients(ResourceId, Conf) -> + ClientOpts = mk_client_opts(ResourceId, Conf), + start_mqtt_clients(ResourceId, Conf, ClientOpts). + +start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> + PoolName = <>, + #{ + pool_size := PoolSize + } = StartConf, Options = [ {name, PoolName}, {pool_size, PoolSize}, {client_opts, ClientOpts} ], - ok = emqx_resource:allocate_resource(ResourceId, egress_pool_name, PoolName), - case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_egress, Options) of + ok = emqx_resource:allocate_resource(ResourceId, pool_name, PoolName), + case emqx_resource_pool:start(PoolName, ?MODULE, Options) of ok -> - {ok, #{ - egress_pool_name => PoolName, - egress_config => emqx_bridge_mqtt_egress:config(Egress) - }}; + {ok, #{pool_name => PoolName}}; {error, {start_pool_failed, _, Reason}} -> {error, Reason} end. -on_stop(ResourceId, _State) -> +on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", connector => ResourceId }), + %% on_stop can be called with State = undefined + StateMap = + case State of + Map when is_map(State) -> + Map; + _ -> + #{} + end, + case maps:get(topic_to_handler_index, StateMap, undefined) of + undefined -> + ok; + TopicToHandlerIndex -> + emqx_topic_index:delete(TopicToHandlerIndex) + end, Allocated = emqx_resource:get_allocated_resources(ResourceId), - ok = stop_ingress(Allocated), - ok = stop_egress(Allocated). + ok = stop_helper(Allocated). -stop_ingress(#{ingress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_ingress(#{}) -> - ok. - -stop_egress(#{egress_pool_name := PoolName}) -> - emqx_resource_pool:stop(PoolName); -stop_egress(#{}) -> - ok. +stop_helper(#{pool_name := PoolName}) -> + emqx_resource_pool:stop(PoolName). on_query( ResourceId, - {send_message, Msg}, - #{egress_pool_name := PoolName, egress_config := Config} + {ChannelId, Msg}, + #{pool_name := PoolName} = State ) -> - ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), - handle_send_result(with_egress_client(PoolName, send, [Msg, Config])); -on_query(ResourceId, {send_message, Msg}, #{}) -> + ?TRACE( + "QUERY", + "send_msg_to_remote_node", + #{ + message => Msg, + connector => ResourceId, + channel_id => ChannelId + } + ), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + handle_send_result(with_egress_client(PoolName, send, [Msg, ChannelConfig])); +on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -187,13 +269,15 @@ on_query(ResourceId, {send_message, Msg}, #{}) -> on_query_async( ResourceId, - {send_message, Msg}, + {ChannelId, Msg}, CallbackIn, - #{egress_pool_name := PoolName, egress_config := Config} + #{pool_name := PoolName} = State ) -> ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}), Callback = {fun on_async_result/2, [CallbackIn]}, - Result = with_egress_client(PoolName, send_async, [Msg, Callback, Config]), + Channels = maps:get(installed_channels, State), + ChannelConfig = maps:get(ChannelId, Channels), + Result = with_egress_client(PoolName, send_async, [Msg, Callback, ChannelConfig]), case Result of ok -> ok; @@ -202,7 +286,7 @@ on_query_async( {error, Reason} -> {error, classify_error(Reason)} end; -on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) -> +on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", connector => ResourceId, @@ -251,7 +335,7 @@ classify_error(Reason) -> {unrecoverable_error, Reason}. on_get_status(_ResourceId, State) -> - Pools = maps:to_list(maps:with([ingress_pool_name, egress_pool_name], State)), + Pools = maps:to_list(maps:with([pool_name], State)), Workers = [{Pool, Worker} || {Pool, PN} <- Pools, {_Name, Worker} <- ecpool:workers(PN)], try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of Statuses -> @@ -261,12 +345,10 @@ on_get_status(_ResourceId, State) -> connecting end. -get_status({Pool, Worker}) -> +get_status({_Pool, Worker}) -> case ecpool_worker:client(Worker) of - {ok, Client} when Pool == ingress_pool_name -> + {ok, Client} -> emqx_bridge_mqtt_ingress:status(Client); - {ok, Client} when Pool == egress_pool_name -> - emqx_bridge_mqtt_egress:status(Client); {error, _} -> disconnected end. @@ -284,30 +366,19 @@ combine_status(Statuses) -> end. mk_ingress_config( - ResourceId, - #{ - ingress := Ingress = #{remote := _}, - server := Server, - hookpoint := HookPoint - } + ChannelId, + IngressChannelConfig, + TopicToHandlerIndex ) -> - Ingress#{ - server => Server, - on_message_received => {?MODULE, on_message_received, [HookPoint, ResourceId]} - }; -mk_ingress_config(ResourceId, #{ingress := #{remote := _}} = Conf) -> - error({no_hookpoint_provided, ResourceId, Conf}); -mk_ingress_config(_ResourceId, #{}) -> - undefined. - -mk_egress_config(#{egress := Egress = #{remote := _}}) -> - Egress; -mk_egress_config(#{}) -> - undefined. + HookPoints = maps:get(hookpoints, IngressChannelConfig, []), + NewConf = IngressChannelConfig#{ + on_message_received => {?MODULE, on_message_received, [HookPoints, ChannelId]}, + ingress_list => [IngressChannelConfig] + }, + emqx_bridge_mqtt_ingress:config(NewConf, ChannelId, TopicToHandlerIndex). mk_client_opts( ResourceId, - ClientScope, Config = #{ server := Server, keepalive := KeepAlive, @@ -327,14 +398,15 @@ mk_client_opts( % A load balancing server (such as haproxy) is often set up before the emqx broker server. % When the load balancing server enables mqtt connection packet inspection, % non-standard mqtt connection packets might be filtered out by LB. - bridge_mode + bridge_mode, + topic_to_handler_index ], Config ), Name = parse_id_to_name(ResourceId), mk_client_opt_password(Options#{ hosts => [HostPort], - clientid => clientid(Name, ClientScope, Config), + clientid => clientid(Name, Config), connect_timeout => 30, keepalive => ms_to_s(KeepAlive), force_ping => true, @@ -357,9 +429,75 @@ mk_client_opt_password(Options) -> ms_to_s(Ms) -> erlang:ceil(Ms / 1000). -clientid(Name, ClientScope, _Conf = #{clientid_prefix := Prefix}) when +clientid(Name, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) andalso Prefix =/= <<>> -> - emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name, ClientScope]); -clientid(Name, ClientScope, _Conf) -> - emqx_bridge_mqtt_lib:clientid_base([Name, ClientScope]). + emqx_bridge_mqtt_lib:clientid_base([Prefix, $:, Name]); +clientid(Name, _Conf) -> + emqx_bridge_mqtt_lib:clientid_base([Name]). + +%% @doc Start an ingress bridge worker. +-spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> + {ok, pid()} | {error, _Reason}. +connect(Options) -> + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ?SLOG(debug, #{ + msg => "ingress_client_starting", + options => emqx_utils:redact(Options) + }), + Name = proplists:get_value(name, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + WorkerId = proplists:get_value(ecpool_worker_id, Options), + ClientOpts = proplists:get_value(client_opts, Options), + case emqtt:start_link(mk_client_opts(Name, WorkerId, ClientOpts)) of + {ok, Pid} -> + connect(Pid, Name); + {error, Reason} = Error -> + ?SLOG(error, #{ + msg => "client_start_failed", + config => emqx_utils:redact(ClientOpts), + reason => Reason + }), + Error + end. + +mk_client_opts( + Name, + WorkerId, + ClientOpts = #{ + clientid := ClientId, + topic_to_handler_index := TopicToHandlerIndex + } +) -> + ClientOpts#{ + clientid := mk_clientid(WorkerId, ClientId), + msg_handler => mk_client_event_handler(Name, TopicToHandlerIndex) + }. + +mk_clientid(WorkerId, ClientId) -> + iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + +mk_client_event_handler(Name, TopicToHandlerIndex) -> + #{ + publish => {fun emqx_bridge_mqtt_ingress:handle_publish/3, [Name, TopicToHandlerIndex]}, + disconnected => {fun ?MODULE:handle_disconnect/1, []} + }. + +-spec connect(pid(), name()) -> + {ok, pid()} | {error, _Reason}. +connect(Pid, Name) -> + case emqtt:connect(Pid) of + {ok, _Props} -> + {ok, Pid}; + {error, Reason} = Error -> + ?SLOG(warning, #{ + msg => "ingress_client_connect_failed", + reason => Reason, + name => Name + }), + _ = catch emqtt:stop(Pid), + Error + end. + +handle_disconnect(_Reason) -> + ok. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 32f9e9295..e863d2a2e 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------- %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +30,10 @@ parse_server/1 ]). +-export([ + connector_examples/1 +]). + -import(emqx_schema, [mk_duration/2]). -import(hoconsc, [mk/2, ref/2]). @@ -61,6 +65,39 @@ fields("config") -> } )} ]; +fields("config_connector") -> + [ + {enable, + mk( + boolean(), + #{ + desc => <<"Enable or disable this connector">>, + default => true + } + )}, + {description, emqx_schema:description_schema()}, + {resource_opts, + mk( + hoconsc:ref(creation_opts), + #{ + required => false, + desc => ?DESC(emqx_resource_schema, "creation_opts") + } + )}, + {pool_size, fun egress_pool_size/1} + % {ingress, + % mk( + % hoconsc:array( + % hoconsc:ref(connector_ingress) + % ), + % #{ + % required => {false, recursively}, + % desc => ?DESC("ingress_desc") + % } + % )} + ] ++ fields("server_configs"); +fields(creation_opts) -> + emqx_connector_schema:resource_opts_fields(); fields("server_configs") -> [ {mode, @@ -131,6 +168,7 @@ fields("server_configs") -> fields("ingress") -> [ {pool_size, fun ingress_pool_size/1}, + %% array {remote, mk( ref(?MODULE, "ingress_remote"), @@ -144,6 +182,22 @@ fields("ingress") -> } )} ]; +fields(connector_ingress) -> + [ + {remote, + mk( + ref(?MODULE, "ingress_remote"), + #{desc => ?DESC("ingress_remote")} + )}, + {local, + mk( + ref(?MODULE, "ingress_local"), + #{ + desc => ?DESC("ingress_local"), + importance => ?IMPORTANCE_HIDDEN + } + )} + ]; fields("ingress_remote") -> [ {topic, @@ -269,7 +323,15 @@ fields("egress_remote") -> desc => ?DESC("payload") } )} - ]. + ]; +fields("get_connector") -> + fields("config_connector"); +fields("post_connector") -> + fields("config_connector"); +fields("put_connector") -> + fields("config_connector"); +fields(What) -> + error({emqx_bridge_mqtt_connector_schema, missing_field_handler, What}). ingress_pool_size(desc) -> ?DESC("ingress_pool_size"); @@ -304,3 +366,6 @@ qos() -> parse_server(Str) -> #{hostname := Host, port := Port} = emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS), {Host, Port}. + +connector_examples(_Method) -> + [#{}]. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl index 2573cad8b..38bdd9665 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl @@ -20,33 +20,16 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). - -export([ config/1, send/3, send_async/4 ]). -%% management APIs --export([ - status/1, - info/1 -]). - --type name() :: term(). -type message() :: emqx_types:message() | map(). -type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}. -type remote_message() :: #mqtt_msg{}. --type option() :: - {name, name()} - %% see `emqtt:option()` - | {client_opts, map()}. - -type egress() :: #{ local => #{ topic => emqx_types:topic() @@ -54,51 +37,6 @@ remote := emqx_bridge_mqtt_msg:msgvars() }. -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "egress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(WorkerId, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name); - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "egress_client_start_failed", - config => emqx_utils:redact(ClientOpts), - reason => Reason - }), - Error - end. - -mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}. - -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -connect(Pid, Name) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "egress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error - end. - -%% - -spec config(map()) -> egress(). config(#{remote := RC = #{}} = Conf) -> @@ -137,25 +75,3 @@ to_remote_msg(Msg = #{}, Remote) -> props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }. - -%% - --spec info(pid()) -> - [{atom(), term()}]. -info(Pid) -> - emqtt:info(Pid). - --spec status(pid()) -> - emqx_resource:resource_status(). -status(Pid) -> - try - case proplists:get_value(socket, info(Pid)) of - Socket when Socket /= undefined -> - connected; - undefined -> - connecting - end - catch - exit:{noproc, _} -> - disconnected - end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index a051ffbd8..d59318a84 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -17,129 +17,188 @@ -module(emqx_bridge_mqtt_ingress). -include_lib("emqx/include/logger.hrl"). - --behaviour(ecpool_worker). - -%% ecpool --export([connect/1]). +-include_lib("emqx/include/emqx_mqtt.hrl"). %% management APIs -export([ status/1, - info/1 + info/1, + subscribe_channel/2, + unsubscribe_channel/4, + config/3 ]). --export([handle_publish/5]). --export([handle_disconnect/1]). +-export([handle_publish/3]). --type name() :: term(). +subscribe_channel(PoolName, ChannelConfig) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + Results = [ + subscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize) + || {{Name, Idx}, Pid} <- Workers + ], + case proplists:get_value(error, Results, ok) of + ok -> + ok; + Error -> + Error + end. --type option() :: - {name, name()} - | {ingress, map()} - %% see `emqtt:option()` - | {client_opts, map()}. +subscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize); + {error, Reason} -> + error({client_not_found, Reason}) + end. --type ingress() :: #{ - server := string(), - remote := #{ - topic := emqx_types:topic(), - qos => emqx_types:qos() - }, - local := emqx_bridge_mqtt_msg:msgvars(), - on_message_received := {module(), atom(), [term()]} -}. - -%% @doc Start an ingress bridge worker. --spec connect([option() | {ecpool_worker_id, pos_integer()}]) -> - {ok, pid()} | {error, _Reason}. -connect(Options) -> - ?SLOG(debug, #{ - msg => "ingress_client_starting", - options => emqx_utils:redact(Options) - }), - Name = proplists:get_value(name, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), - Ingress = config(proplists:get_value(ingress, Options), Name), - ClientOpts = proplists:get_value(client_opts, Options), - case emqtt:start_link(mk_client_opts(Name, WorkerId, Ingress, ClientOpts)) of - {ok, Pid} -> - connect(Pid, Name, Ingress); +subscribe_channel_helper(Client, Name, Ingress, WorkerIdx, PoolSize) -> + IngressList = maps:get(ingress_list, Ingress, []), + SubscribeResults = subscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name + ), + %% Find error if any using proplists:get_value/2 + case proplists:get_value(error, SubscribeResults, ok) of + ok -> + ok; {error, Reason} = Error -> ?SLOG(error, #{ - msg => "client_start_failed", - config => emqx_utils:redact(ClientOpts), + msg => "ingress_client_subscribe_failed", + ingress => Ingress, + name => Name, reason => Reason }), Error end. -mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) -> - ClientOpts#{ - clientid := mk_clientid(WorkerId, ClientId), - msg_handler => mk_client_event_handler(Name, Ingress) - }. +subscribe_remote_topics(Pid, IngressList, WorkerIdx, PoolSize, Name) -> + [subscribe_remote_topic(Pid, Ingress, WorkerIdx, PoolSize, Name) || Ingress <- IngressList]. -mk_clientid(WorkerId, ClientId) -> - emqx_bridge_mqtt_lib:bytes23(ClientId, WorkerId). - -mk_client_event_handler(Name, Ingress = #{}) -> - IngressVars = maps:with([server], Ingress), - OnMessage = maps:get(on_message_received, Ingress, undefined), - LocalPublish = - case Ingress of - #{local := Local = #{topic := _}} -> - Local; - #{} -> - undefined - end, - #{ - publish => {fun ?MODULE:handle_publish/5, [Name, OnMessage, LocalPublish, IngressVars]}, - disconnected => {fun ?MODULE:handle_disconnect/1, []} - }. - --spec connect(pid(), name(), ingress()) -> - {ok, pid()} | {error, _Reason}. -connect(Pid, Name, Ingress) -> - case emqtt:connect(Pid) of - {ok, _Props} -> - case subscribe_remote_topic(Pid, Ingress) of - {ok, _, _RCs} -> - {ok, Pid}; - {error, Reason} = Error -> - ?SLOG(error, #{ - msg => "ingress_client_subscribe_failed", - ingress => Ingress, - name => Name, - reason => Reason - }), - _ = catch emqtt:stop(Pid), - Error - end; - {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => "ingress_client_connect_failed", - reason => Reason, - name => Name - }), - _ = catch emqtt:stop(Pid), - Error +subscribe_remote_topic( + Pid, #{remote := #{topic := RemoteTopic, qos := QoS}} = _Remote, WorkerIdx, PoolSize, Name +) -> + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _LogWarn = true) of + true -> + emqtt:subscribe(Pid, RemoteTopic, QoS); + false -> + ok end. -subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) -> - emqtt:subscribe(Pid, RemoteTopic, QoS). +should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, LogWarn) -> + IsFirstWorker = WorkerIdx == 1, + case emqx_topic:parse(RemoteTopic) of + {#share{} = _Filter, _SubOpts} -> + % NOTE: this is shared subscription, many workers may subscribe + true; + {_Filter, #{}} when PoolSize > 1, IsFirstWorker, LogWarn -> + % NOTE: this is regular subscription, only one worker should subscribe + ?SLOG(warning, #{ + msg => "mqtt_pool_size_ignored", + connector => Name, + reason => + "Remote topic filter is not a shared subscription, " + "only a single connection will be used from the connection pool", + config_pool_size => PoolSize, + pool_size => PoolSize + }), + IsFirstWorker; + {_Filter, #{}} -> + % NOTE: this is regular subscription, only one worker should subscribe + IsFirstWorker + end. -%% +unsubscribe_channel(PoolName, ChannelConfig, ChannelId, TopicToHandlerIndex) -> + Workers = ecpool:workers(PoolName), + PoolSize = length(Workers), + _ = [ + unsubscribe_channel(Pid, Name, ChannelConfig, Idx, PoolSize, ChannelId, TopicToHandlerIndex) + || {{Name, Idx}, Pid} <- Workers + ], + ok. --spec config(map(), name()) -> - ingress(). -config(#{remote := RC, local := LC} = Conf, BridgeName) -> - Conf#{ +unsubscribe_channel(WorkerPid, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex) -> + case ecpool_worker:client(WorkerPid) of + {ok, Client} -> + unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex + ); + {error, Reason} -> + error({client_not_found, Reason}) + end. + +unsubscribe_channel_helper( + Client, Name, Ingress, WorkerIdx, PoolSize, ChannelId, TopicToHandlerIndex +) -> + IngressList = maps:get(ingress_list, Ingress, []), + unsubscribe_remote_topics( + Client, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ). + +unsubscribe_remote_topics( + Pid, IngressList, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex +) -> + [ + unsubscribe_remote_topic( + Pid, Ingress, WorkerIdx, PoolSize, Name, ChannelId, TopicToHandlerIndex + ) + || Ingress <- IngressList + ]. + +unsubscribe_remote_topic( + Pid, + #{remote := #{topic := RemoteTopic}} = _Remote, + WorkerIdx, + PoolSize, + Name, + ChannelId, + TopicToHandlerIndex +) -> + emqx_topic_index:delete(RemoteTopic, ChannelId, TopicToHandlerIndex), + case should_subscribe(RemoteTopic, WorkerIdx, PoolSize, Name, _NoWarn = false) of + true -> + case emqtt:unsubscribe(Pid, RemoteTopic) of + {ok, _Properties, _ReasonCodes} -> + ok; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "unsubscribe_mqtt_topic_failed", + channel_id => Name, + reason => Reason + }), + ok + end; + false -> + ok + end. + +config(#{ingress_list := IngressList} = Conf, Name, TopicToHandlerIndex) -> + NewIngressList = [ + fix_remote_config(Ingress, Name, TopicToHandlerIndex, Conf) + || Ingress <- IngressList + ], + Conf#{ingress_list => NewIngressList}. + +fix_remote_config(#{remote := RC, local := LC}, BridgeName, TopicToHandlerIndex, Conf) -> + FixedConf = Conf#{ remote => parse_remote(RC, BridgeName), local => emqx_bridge_mqtt_msg:parse(LC) - }. + }, + insert_to_topic_to_handler_index(FixedConf, TopicToHandlerIndex, BridgeName), + FixedConf. -parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> +insert_to_topic_to_handler_index( + #{remote := #{topic := Topic}} = Conf, TopicToHandlerIndex, BridgeName +) -> + TopicPattern = + case emqx_topic:parse(Topic) of + {#share{group = _Group, topic = TP}, _} -> + TP; + _ -> + Topic + end, + emqx_topic_index:insert(TopicPattern, BridgeName, Conf, TopicToHandlerIndex). + +parse_remote(#{qos := QoSIn} = Remote, BridgeName) -> QoS = downgrade_ingress_qos(QoSIn), case QoS of QoSIn -> @@ -152,7 +211,7 @@ parse_remote(#{qos := QoSIn} = Conf, BridgeName) -> name => BridgeName }) end, - Conf#{qos => QoS}. + Remote#{qos => QoS}. downgrade_ingress_qos(2) -> 1; @@ -183,17 +242,39 @@ status(Pid) -> %% -handle_publish(#{properties := Props} = MsgIn, Name, OnMessage, LocalPublish, IngressVars) -> - Msg = import_msg(MsgIn, IngressVars), +handle_publish( + #{properties := Props, topic := Topic} = MsgIn, + Name, + TopicToHandlerIndex +) -> ?SLOG(debug, #{ msg => "ingress_publish_local", - message => Msg, + message => MsgIn, name => Name }), - maybe_on_message_received(Msg, OnMessage), - maybe_publish_local(Msg, LocalPublish, Props). + Matches = emqx_topic_index:matches(Topic, TopicToHandlerIndex, []), + lists:foreach( + fun(Match) -> + handle_match(TopicToHandlerIndex, Match, MsgIn, Name, Props) + end, + Matches + ), + ok. -handle_disconnect(_Reason) -> +handle_match( + TopicToHandlerIndex, + Match, + MsgIn, + _Name, + Props +) -> + [ChannelConfig] = emqx_topic_index:get_record(Match, TopicToHandlerIndex), + #{on_message_received := OnMessage} = ChannelConfig, + Msg = import_msg(MsgIn, ChannelConfig), + + maybe_on_message_received(Msg, OnMessage), + LocalPublish = maps:get(local, ChannelConfig, undefined), + _ = maybe_publish_local(Msg, LocalPublish, Props), ok. maybe_on_message_received(Msg, {Mod, Func, Args}) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl new file mode 100644 index 000000000..6bcdc611b --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -0,0 +1,221 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_mqtt_pubsub_action_info). + +-behaviour(emqx_action_info). + +-export([ + bridge_v1_type_name/0, + action_type_name/0, + connector_type_name/0, + schema_module/0, + bridge_v1_config_to_connector_config/1, + bridge_v1_config_to_action_config/2, + connector_action_config_to_bridge_v1_config/2, + is_source/0 +]). + +bridge_v1_type_name() -> mqtt. + +action_type_name() -> mqtt. + +connector_type_name() -> mqtt. + +schema_module() -> emqx_bridge_mqtt_pubsub_schema. + +is_source() -> true. + +bridge_v1_config_to_connector_config(Config) -> + %% Transform the egress part to mqtt_publisher connector config + SimplifiedConfig = check_and_simplify_bridge_v1_config(Config), + ConnectorConfigMap = make_connector_config_from_bridge_v1_config(SimplifiedConfig), + {mqtt, ConnectorConfigMap}. + +make_connector_config_from_bridge_v1_config(Config) -> + ConnectorConfigSchema = emqx_bridge_mqtt_connector_schema:fields("config_connector"), + ConnectorTopFields = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ConnectorConfigSchema + ], + ConnectorConfigMap = maps:with(ConnectorTopFields, Config), + ResourceOptsSchema = emqx_bridge_mqtt_connector_schema:fields(creation_opts), + ResourceOptsTopFields = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ResourceOptsSchema + ], + ResourceOptsMap = maps:get(<<"resource_opts">>, ConnectorConfigMap, #{}), + ResourceOptsMap2 = maps:with(ResourceOptsTopFields, ResourceOptsMap), + ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), + IngressMap0 = maps:get(<<"ingress">>, Config, #{}), + EgressMap = maps:get(<<"egress">>, Config, #{}), + % %% Move pool_size to the top level + PoolSizeIngress = maps:get(<<"pool_size">>, IngressMap0, undefined), + PoolSize = + case PoolSizeIngress of + undefined -> + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + maps:get(<<"pool_size">>, EgressMap, DefaultPoolSize); + _ -> + PoolSizeIngress + end, + % IngressMap1 = maps:remove(<<"pool_size">>, IngressMap0), + %% Remove ingress part from the config + ConnectorConfigMap3 = maps:remove(<<"ingress">>, ConnectorConfigMap2), + %% Remove egress part from the config + ConnectorConfigMap4 = maps:remove(<<"egress">>, ConnectorConfigMap3), + ConnectorConfigMap5 = maps:put(<<"pool_size">>, PoolSize, ConnectorConfigMap4), + % ConnectorConfigMap4 = + % case IngressMap1 =:= #{} of + % true -> + % ConnectorConfigMap3; + % false -> + % maps:put(<<"ingress">>, [IngressMap1], ConnectorConfigMap3) + % end, + ConnectorConfigMap5. + +bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> + SimplifiedConfig = check_and_simplify_bridge_v1_config(BridgeV1Config), + bridge_v1_config_to_action_config_helper( + SimplifiedConfig, ConnectorName + ). + +bridge_v1_config_to_action_config_helper( + #{ + <<"egress">> := EgressMap0 + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_publisher_action"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), + LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), + EgressMap1 = maps:remove(<<"local">>, EgressMap0), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), + ConfigMap3 = + case LocalTopic of + undefined -> + ConfigMap2; + _ -> + maps:put(<<"local_topic">>, LocalTopic, ConfigMap2) + end, + {action, mqtt, ConfigMap3}; +bridge_v1_config_to_action_config_helper( + #{ + <<"ingress">> := IngressMap + } = Config, + ConnectorName +) -> + %% Transform the egress part to mqtt_publisher connector config + SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ConfigMap1 = general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields + ), + IngressMap1 = maps:remove(<<"pool_size">>, IngressMap), + %% Add parameters field (Egress map) to the action config + ConfigMap2 = maps:put(<<"parameters">>, IngressMap1, ConfigMap1), + {source, mqtt, ConfigMap2}; +bridge_v1_config_to_action_config_helper( + _Config, + _ConnectorName +) -> + error({incompatible_bridge_v1, no_matching_action_or_source}). + +general_action_conf_map_from_bridge_v1_config( + Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields +) -> + ShemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- SchemaFields + ], + ActionConfig0 = maps:with(ShemaFieldsNames, Config), + ResourceOptsSchemaFieldsNames = [ + erlang:atom_to_binary(FieldName, utf8) + || {FieldName, _} <- ResourceOptsSchemaFields + ], + ResourceOptsMap = maps:get(<<"resource_opts">>, ActionConfig0, #{}), + ResourceOptsMap2 = maps:with(ResourceOptsSchemaFieldsNames, ResourceOptsMap), + %% Only put resource_opts if the original config has it + ActionConfig1 = + case maps:is_key(<<"resource_opts">>, ActionConfig0) of + true -> + maps:put(<<"resource_opts">>, ResourceOptsMap2, ActionConfig0); + false -> + ActionConfig0 + end, + ActionConfig2 = maps:put(<<"connector">>, ConnectorName, ActionConfig1), + ActionConfig2. + +check_and_simplify_bridge_v1_config( + #{ + <<"egress">> := EgressMap + } = Config +) when map_size(EgressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"egress">>, Config)); +check_and_simplify_bridge_v1_config( + #{ + <<"ingress">> := IngressMap + } = Config +) when map_size(IngressMap) =:= 0 -> + check_and_simplify_bridge_v1_config(maps:remove(<<"ingress">>, Config)); +check_and_simplify_bridge_v1_config(#{ + <<"egress">> := _EGressMap, + <<"ingress">> := _InGressMap +}) -> + %% We should crash beacuse we don't support upgrading when ingress and egress exist at the same time + error( + {unsupported_config, + <<"Upgrade not supported when ingress and egress exist in the same MQTT bridge. Please divide the egress and ingress part to separate bridges in the configuration.">>} + ); +check_and_simplify_bridge_v1_config(SimplifiedConfig) -> + SimplifiedConfig. + +connector_action_config_to_bridge_v1_config( + ConnectorConfig, ActionConfig +) -> + Params = maps:get(<<"parameters">>, ActionConfig, #{}), + ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), + ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), + ResourceOpts = maps:merge(ResourceOptsConnector, ResourceOptsAction), + %% Check the direction of the action + Direction = + case maps:get(<<"remote">>, Params) of + #{<<"retain">> := _} -> + %% Only source has retain + <<"publisher">>; + _ -> + <<"subscriber">> + end, + Parms2 = maps:remove(<<"direction">>, Params), + DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), + PoolSize = maps:get(<<"pool_size">>, ConnectorConfig, DefaultPoolSize), + Parms3 = maps:put(<<"pool_size">>, PoolSize, Parms2), + ConnectorConfig2 = maps:remove(<<"pool_size">>, ConnectorConfig), + LocalTopic = maps:get(<<"local_topic">>, ActionConfig, undefined), + BridgeV1Conf0 = + case {Direction, LocalTopic} of + {<<"publisher">>, undefined} -> + #{<<"egress">> => Parms3}; + {<<"publisher">>, LocalT} -> + #{ + <<"egress">> => Parms3, + <<"local">> => + #{ + <<"topic">> => LocalT + } + }; + {<<"subscriber">>, _} -> + #{<<"ingress">> => Parms3} + end, + BridgeV1Conf1 = maps:merge(BridgeV1Conf0, ConnectorConfig2), + BridgeV1Conf2 = BridgeV1Conf1#{ + <<"resource_opts">> => ResourceOpts + }, + BridgeV1Conf2. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl new file mode 100644 index 000000000..2aba6e8ea --- /dev/null +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -0,0 +1,129 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_pubsub_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, ref/2]). + +-export([roots/0, fields/1, desc/1, namespace/0]). + +-export([ + bridge_v2_examples/1, + conn_bridge_examples/1 +]). + +%%====================================================================================== +%% Hocon Schema Definitions +namespace() -> "bridge_mqtt_publisher". + +roots() -> []. + +fields(action) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_publisher_action")), + #{ + desc => <<"MQTT Publisher Action Config">>, + required => false + } + )}; +fields("mqtt_publisher_action") -> + emqx_bridge_v2_schema:make_producer_action_schema( + hoconsc:mk( + hoconsc:ref(?MODULE, action_parameters), + #{ + required => true, + desc => ?DESC("action_parameters") + } + ) + ); +fields(action_parameters) -> + Fields0 = emqx_bridge_mqtt_connector_schema:fields("egress"), + Fields1 = proplists:delete(pool_size, Fields0), + Fields2 = proplists:delete(local, Fields1), + Fields2; +fields(source) -> + {mqtt, + mk( + hoconsc:map(name, ref(?MODULE, "mqtt_subscriber_source")), + #{ + desc => <<"MQTT Subscriber Source Config">>, + required => false + } + )}; +fields("mqtt_subscriber_source") -> + emqx_bridge_v2_schema:make_consumer_action_schema( + hoconsc:mk( + hoconsc:ref(?MODULE, ingress_parameters), + #{ + required => true, + desc => ?DESC("source_parameters") + } + ) + ); +fields(ingress_parameters) -> + Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), + Fields1 = proplists:delete(pool_size, Fields0), + Fields1; +fields("resource_opts") -> + UnsupportedOpts = [enable_batch, batch_size, batch_time], + lists:filter( + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, + emqx_resource_schema:fields("creation_opts") + ); +fields("get_connector") -> + emqx_bridge_mqtt_connector_schema:fields("config_connector"); +fields("get_bridge_v2") -> + fields("mqtt_publisher_action"); +fields("post_bridge_v2") -> + fields("mqtt_publisher_action"); +fields("put_bridge_v2") -> + fields("mqtt_publisher_action"); +fields(What) -> + error({emqx_bridge_mqtt_pubsub_schema, missing_field_handler, What}). +%% v2: api schema +%% The parameter equls to +%% `get_bridge_v2`, `post_bridge_v2`, `put_bridge_v2` from emqx_bridge_v2_schema:api_schema/1 +%% `get_connector`, `post_connector`, `put_connector` from emqx_connector_schema:api_schema/1 +%%-------------------------------------------------------------------- +%% v1/v2 + +desc("config") -> + ?DESC("desc_config"); +desc("resource_opts") -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; +desc("config_connector") -> + ?DESC("desc_config"); +desc("http_action") -> + ?DESC("desc_config"); +desc("parameters_opts") -> + ?DESC("config_parameters_opts"); +desc(_) -> + undefined. + +bridge_v2_examples(_Method) -> + [ + #{} + ]. + +conn_bridge_examples(_Method) -> + [ + #{} + ]. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6d1ff0915..bd3fb68de 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -238,7 +238,8 @@ t_conf_bridge_authn_passfile(Config) -> post, uri(["bridges"]), ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ - <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + <<"name">> => <<"t_conf_bridge_authn_no_passfile">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} } ), ?assertMatch({match, _}, re:run(Reason, <<"failed_to_read_secret_file">>)). @@ -397,32 +398,25 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), ok. -t_mqtt_egress_bridge_ignores_clean_start(_) -> +t_mqtt_egress_bridge_warns_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), - BridgeID = create_bridge( - ?SERVER_CONF#{ - <<"name">> => BridgeName, - <<"egress">> => ?EGRESS_CONF, - <<"clean_start">> => false - } - ), + Action = fun() -> + BridgeID = create_bridge( + ?SERVER_CONF#{ + <<"name">> => BridgeName, + <<"egress">> => ?EGRESS_CONF, + <<"clean_start">> => false + } + ), - ResourceID = emqx_bridge_resource:resource_id(BridgeID), - {ok, _Group, #{state := #{egress_pool_name := EgressPoolName}}} = - emqx_resource_manager:lookup_cached(ResourceID), - ClientInfo = ecpool:pick_and_do( - EgressPoolName, - {emqx_bridge_mqtt_egress, info, []}, - no_handover + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []) + end, + ?wait_async_action( + Action(), + #{?snk_kind := mqtt_clean_start_egress_action_warning}, + 10000 ), - ?assertMatch( - #{clean_start := true}, - maps:from_list(ClientInfo) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), - ok. t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index a58a1ef3d..f85109080 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -51,11 +51,11 @@ -export([parse_url/1]). --callback connector_config(ParsedConfig) -> +-callback connector_config(ParsedConfig, Context) -> ParsedConfig when - ParsedConfig :: #{atom() => any()}. --optional_callbacks([connector_config/1]). + ParsedConfig :: #{atom() => any()}, Context :: #{atom() => any()}. +-optional_callbacks([connector_config/2]). -if(?EMQX_RELEASE_EDITION == ee). connector_to_resource_type(ConnectorType) -> @@ -81,6 +81,10 @@ connector_impl_module(_ConnectorType) -> connector_to_resource_type_ce(http) -> emqx_bridge_http_connector; +connector_to_resource_type_ce(mqtt) -> + emqx_bridge_mqtt_connector; +% connector_to_resource_type_ce(mqtt_subscriber) -> +% emqx_bridge_mqtt_subscriber_connector; connector_to_resource_type_ce(ConnectorType) -> error({no_bridge_v2, ConnectorType}). @@ -276,6 +280,12 @@ remove(Type, Name, _Conf, _Opts) -> emqx_resource:remove_local(resource_id(Type, Name)). %% convert connector configs to what the connector modules want +parse_confs( + <<"mqtt">> = Type, + Name, + Conf +) -> + insert_hookpoints(Type, Name, Conf); parse_confs( <<"http">>, _Name, @@ -307,6 +317,13 @@ parse_confs( parse_confs(ConnectorType, Name, Config) -> connector_config(ConnectorType, Name, Config). +insert_hookpoints(Type, Name, Conf) -> + BId = emqx_bridge_resource:bridge_id(Type, Name), + BridgeHookpoint = emqx_bridge_resource:bridge_hookpoint(BId), + ConnectorHookpoint = connector_hookpoint(BId), + HookPoints = [BridgeHookpoint, ConnectorHookpoint], + Conf#{hookpoints => HookPoints}. + connector_config(ConnectorType, Name, Config) -> Mod = connector_impl_module(ConnectorType), case erlang:function_exported(Mod, connector_config, 2) of diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index b043ebacd..74b92c165 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -90,7 +90,9 @@ api_schemas(Method) -> [ %% We need to map the `type' field of a request (binary) to a %% connector schema module. - api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector") + api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector"), + % api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt_subscriber">>, Method ++ "_connector"), + api_ref(emqx_bridge_mqtt_connector_schema, <<"mqtt">>, Method ++ "_connector") ]. api_ref(Module, Type, Method) -> @@ -110,10 +112,11 @@ examples(Method) -> -if(?EMQX_RELEASE_EDITION == ee). schema_modules() -> - [emqx_bridge_http_schema] ++ emqx_connector_ee_schema:schema_modules(). + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema] ++ + emqx_connector_ee_schema:schema_modules(). -else. schema_modules() -> - [emqx_bridge_http_schema]. + [emqx_bridge_http_schema, emqx_bridge_mqtt_connector_schema]. -endif. %% @doc Return old bridge(v1) and/or connector(v2) type @@ -136,6 +139,8 @@ connector_type_to_bridge_types(influxdb) -> [influxdb, influxdb_api_v1, influxdb_api_v2]; connector_type_to_bridge_types(mysql) -> [mysql]; +connector_type_to_bridge_types(mqtt) -> + [mqtt]; connector_type_to_bridge_types(pgsql) -> [pgsql]; connector_type_to_bridge_types(redis) -> @@ -151,7 +156,8 @@ connector_type_to_bridge_types(iotdb) -> connector_type_to_bridge_types(elasticsearch) -> [elasticsearch]. -actions_config_name() -> <<"actions">>. +actions_config_name(action) -> <<"actions">>; +actions_config_name(source) -> <<"sources">>. has_connector_field(BridgeConf, ConnectorFields) -> lists:any( @@ -185,40 +191,58 @@ bridge_configs_to_transform( end. split_bridge_to_connector_and_action( - {ConnectorsMap, {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig}} + { + {ConnectorsMap, OrgConnectorType}, + {BridgeType, BridgeName, BridgeV1Conf, ConnectorFields, PreviousRawConfig} + } ) -> - ConnectorMap = + {ConnectorMap, ConnectorType} = case emqx_action_info:has_custom_bridge_v1_config_to_connector_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_connector_config( - BridgeType, BridgeV1Conf - ); + case + emqx_action_info:bridge_v1_config_to_connector_config( + BridgeType, BridgeV1Conf + ) + of + {ConType, ConMap} -> + {ConMap, ConType}; + ConMap -> + {ConMap, OrgConnectorType} + end; false -> %% We do an automatic transformation to get the connector config %% if the callback is not defined. %% Get connector fields from bridge config - lists:foldl( - fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> - ConnectorFieldNameBin = to_bin(ConnectorFieldName), - case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of - true -> - PrevFieldConfig = - maybe_project_to_connector_resource_opts( + NewCConMap = + lists:foldl( + fun({ConnectorFieldName, _Spec}, ToTransformSoFar) -> + ConnectorFieldNameBin = to_bin(ConnectorFieldName), + case maps:is_key(ConnectorFieldNameBin, BridgeV1Conf) of + true -> + PrevFieldConfig = + maybe_project_to_connector_resource_opts( + ConnectorFieldNameBin, + maps:get(ConnectorFieldNameBin, BridgeV1Conf) + ), + NewToTransform0 = maps:put( ConnectorFieldNameBin, - maps:get(ConnectorFieldNameBin, BridgeV1Conf) + PrevFieldConfig, + ToTransformSoFar ), - maps:put( - ConnectorFieldNameBin, - PrevFieldConfig, + NewToTransform1 = maps:put( + to_bin(ConnectorFieldName), + maps:get(to_bin(ConnectorFieldName), BridgeV1Conf), + NewToTransform0 + ), + NewToTransform1; + false -> ToTransformSoFar - ); - false -> - ToTransformSoFar - end - end, - #{}, - ConnectorFields - ) + end + end, + #{}, + ConnectorFields + ), + {NewCConMap, OrgConnectorType} end, %% Generate a connector name, if needed. Avoid doing so if there was a previous config. ConnectorName = @@ -226,18 +250,29 @@ split_bridge_to_connector_and_action( #{<<"connector">> := ConnectorName0} -> ConnectorName0; _ -> generate_connector_name(ConnectorsMap, BridgeName, 0) end, - ActionMap = + OrgActionType = emqx_action_info:bridge_v1_type_to_action_type(BridgeType), + {ActionMap, ActionType, ActionOrSource} = case emqx_action_info:has_custom_bridge_v1_config_to_action_config(BridgeType) of true -> - emqx_action_info:bridge_v1_config_to_action_config( - BridgeType, BridgeV1Conf, ConnectorName - ); + case + emqx_action_info:bridge_v1_config_to_action_config( + BridgeType, BridgeV1Conf, ConnectorName + ) + of + {ActionOrSource0, ActionType0, ActionMap0} -> + {ActionMap0, ActionType0, ActionOrSource0}; + ActionMap0 -> + {ActionMap0, OrgActionType, action} + end; false -> - transform_bridge_v1_config_to_action_config( - BridgeV1Conf, ConnectorName, ConnectorFields - ) + ActionMap0 = + transform_bridge_v1_config_to_action_config( + BridgeV1Conf, ConnectorName, ConnectorFields + ), + {ActionMap0, OrgActionType} end, - {BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}. + {BridgeType, BridgeName, ActionMap, ActionType, ActionOrSource, ConnectorName, ConnectorMap, + ConnectorType}. maybe_project_to_connector_resource_opts(<<"resource_opts">>, OldResourceOpts) -> project_to_connector_resource_opts(OldResourceOpts); @@ -307,9 +342,9 @@ generate_connector_name(ConnectorsMap, BridgeName, Attempt) -> ConnectorNameList = case Attempt of 0 -> - io_lib:format("connector_~s", [BridgeName]); + io_lib:format("~s", [BridgeName]); _ -> - io_lib:format("connector_~s_~p", [BridgeName, Attempt + 1]) + io_lib:format("~s_~p", [BridgeName, Attempt + 1]) end, ConnectorName = iolist_to_binary(ConnectorNameList), case maps:is_key(ConnectorName, ConnectorsMap) of @@ -340,7 +375,10 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), ConnectorsWithTypeMap = maps:get(to_bin(ConnectorType), ConnectorsConfMap, #{}), BridgeConfigsToTransformWithConnectorConf = lists:zip( - lists:duplicate(length(BridgeConfigsToTransform), ConnectorsWithTypeMap), + lists:duplicate( + length(BridgeConfigsToTransform), + {ConnectorsWithTypeMap, ConnectorType} + ), BridgeConfigsToTransform ), ActionConnectorTuples = lists:map( @@ -349,10 +387,14 @@ transform_old_style_bridges_to_connector_and_actions_of_type( ), %% Add connectors and actions and remove bridges lists:foldl( - fun({BridgeType, BridgeName, ActionMap, ConnectorName, ConnectorMap}, RawConfigSoFar) -> + fun( + {BridgeType, BridgeName, ActionMap, NewActionType, ActionOrSource, ConnectorName, + ConnectorMap, NewConnectorType}, + RawConfigSoFar + ) -> %% Add connector RawConfigSoFar1 = emqx_utils_maps:deep_put( - [<<"connectors">>, to_bin(ConnectorType), ConnectorName], + [<<"connectors">>, to_bin(NewConnectorType), ConnectorName], RawConfigSoFar, ConnectorMap ), @@ -362,12 +404,21 @@ transform_old_style_bridges_to_connector_and_actions_of_type( RawConfigSoFar1 ), %% Add action - ActionType = emqx_action_info:bridge_v1_type_to_action_type(to_bin(BridgeType)), - RawConfigSoFar3 = emqx_utils_maps:deep_put( - [actions_config_name(), to_bin(ActionType), BridgeName], - RawConfigSoFar2, - ActionMap - ), + RawConfigSoFar3 = + case ActionMap of + none -> + RawConfigSoFar2; + _ -> + emqx_utils_maps:deep_put( + [ + actions_config_name(ActionOrSource), + to_bin(NewActionType), + BridgeName + ], + RawConfigSoFar2, + ActionMap + ) + end, RawConfigSoFar3 end, RawConfig, @@ -454,7 +505,23 @@ fields(connectors) -> desc => <<"HTTP Connector Config">>, required => false } + )}, + {mqtt, + mk( + hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + #{ + desc => <<"MQTT Publisher Connector Config">>, + required => false + } )} + % {mqtt_subscriber, + % mk( + % hoconsc:map(name, ref(emqx_bridge_mqtt_connector_schema, "config_connector")), + % #{ + % desc => <<"MQTT Subscriber Connector Config">>, + % required => false + % } + % )} ] ++ enterprise_fields_connectors(); fields("node_status") -> [ From 145ed2e6320015025ec5542c4912e8b9a4e6de17 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 15:34:23 +0100 Subject: [PATCH 26/62] fix: elvis style error --- .../src/emqx_bridge_mqtt_pubsub_action_info.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 6bcdc611b..365af1335 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -171,8 +171,11 @@ check_and_simplify_bridge_v1_config(#{ }) -> %% We should crash beacuse we don't support upgrading when ingress and egress exist at the same time error( - {unsupported_config, - <<"Upgrade not supported when ingress and egress exist in the same MQTT bridge. Please divide the egress and ingress part to separate bridges in the configuration.">>} + {unsupported_config, << + "Upgrade not supported when ingress and egress exist in the " + "same MQTT bridge. Please divide the egress and ingress part " + "to separate bridges in the configuration." + >>} ); check_and_simplify_bridge_v1_config(SimplifiedConfig) -> SimplifiedConfig. From 886ed55374a110e7d22e28c8a161b76a44e6304c Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 15:55:38 +0100 Subject: [PATCH 27/62] fix: don't call non-existing function --- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 1bee2c92e..6f9465cb0 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -234,7 +234,7 @@ on_stop(ResourceId, State) -> undefined -> ok; TopicToHandlerIndex -> - emqx_topic_index:delete(TopicToHandlerIndex) + ets:delete(TopicToHandlerIndex) end, Allocated = emqx_resource:get_allocated_resources(ResourceId), ok = stop_helper(Allocated). From 2ecc775fb7202a4145824ba6820bc8b28cae40c9 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 14 Dec 2023 16:04:02 +0100 Subject: [PATCH 28/62] style: remove commented out code and fix copyright headers --- .../emqx_bridge_mqtt_pubsub_action_info.erl | 24 +++++++++++-------- .../src/emqx_bridge_mqtt_schema.erl | 1 + 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 365af1335..0a1eefd82 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_bridge_mqtt_pubsub_action_info). @@ -50,7 +62,7 @@ make_connector_config_from_bridge_v1_config(Config) -> ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), IngressMap0 = maps:get(<<"ingress">>, Config, #{}), EgressMap = maps:get(<<"egress">>, Config, #{}), - % %% Move pool_size to the top level + %% Move pool_size to the top level PoolSizeIngress = maps:get(<<"pool_size">>, IngressMap0, undefined), PoolSize = case PoolSizeIngress of @@ -60,19 +72,11 @@ make_connector_config_from_bridge_v1_config(Config) -> _ -> PoolSizeIngress end, - % IngressMap1 = maps:remove(<<"pool_size">>, IngressMap0), %% Remove ingress part from the config ConnectorConfigMap3 = maps:remove(<<"ingress">>, ConnectorConfigMap2), %% Remove egress part from the config ConnectorConfigMap4 = maps:remove(<<"egress">>, ConnectorConfigMap3), ConnectorConfigMap5 = maps:put(<<"pool_size">>, PoolSize, ConnectorConfigMap4), - % ConnectorConfigMap4 = - % case IngressMap1 =:= #{} of - % true -> - % ConnectorConfigMap3; - % false -> - % maps:put(<<"ingress">>, [IngressMap1], ConnectorConfigMap3) - % end, ConnectorConfigMap5. bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl index a312dfaa9..37d0b4f03 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl @@ -13,6 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- + -module(emqx_bridge_mqtt_schema). -include_lib("typerefl/include/types.hrl"). From 139da6d720d0f9941865ea936e2e577951e41bf6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:26:05 -0300 Subject: [PATCH 29/62] fix: don't double-write the transformed config; return a triplet in all cases --- .../src/schema/emqx_connector_schema.erl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 74b92c165..b51be53ed 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -224,17 +224,11 @@ split_bridge_to_connector_and_action( ConnectorFieldNameBin, maps:get(ConnectorFieldNameBin, BridgeV1Conf) ), - NewToTransform0 = maps:put( + maps:put( ConnectorFieldNameBin, PrevFieldConfig, ToTransformSoFar - ), - NewToTransform1 = maps:put( - to_bin(ConnectorFieldName), - maps:get(to_bin(ConnectorFieldName), BridgeV1Conf), - NewToTransform0 - ), - NewToTransform1; + ); false -> ToTransformSoFar end @@ -269,7 +263,7 @@ split_bridge_to_connector_and_action( transform_bridge_v1_config_to_action_config( BridgeV1Conf, ConnectorName, ConnectorFields ), - {ActionMap0, OrgActionType} + {ActionMap0, OrgActionType, action} end, {BridgeType, BridgeName, ActionMap, ActionType, ActionOrSource, ConnectorName, ConnectorMap, ConnectorType}. From 7befe898d09fd9546731f7ec6a226c8e4d8300f8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:44:19 -0300 Subject: [PATCH 30/62] fix(mqtt_bridge): fix schema --- .../src/emqx_bridge_mqtt_connector_schema.erl | 13 ++------- .../emqx_bridge_mqtt_pubsub_action_info.erl | 29 +++++++++++-------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index e863d2a2e..ba4373313 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -76,14 +76,6 @@ fields("config_connector") -> } )}, {description, emqx_schema:description_schema()}, - {resource_opts, - mk( - hoconsc:ref(creation_opts), - #{ - required => false, - desc => ?DESC(emqx_resource_schema, "creation_opts") - } - )}, {pool_size, fun egress_pool_size/1} % {ingress, % mk( @@ -95,8 +87,9 @@ fields("config_connector") -> % desc => ?DESC("ingress_desc") % } % )} - ] ++ fields("server_configs"); -fields(creation_opts) -> + ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + fields("server_configs"); +fields(resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields("server_configs") -> [ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 0a1eefd82..de39fc9b4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -52,13 +52,8 @@ make_connector_config_from_bridge_v1_config(Config) -> || {FieldName, _} <- ConnectorConfigSchema ], ConnectorConfigMap = maps:with(ConnectorTopFields, Config), - ResourceOptsSchema = emqx_bridge_mqtt_connector_schema:fields(creation_opts), - ResourceOptsTopFields = [ - erlang:atom_to_binary(FieldName, utf8) - || {FieldName, _} <- ResourceOptsSchema - ], ResourceOptsMap = maps:get(<<"resource_opts">>, ConnectorConfigMap, #{}), - ResourceOptsMap2 = maps:with(ResourceOptsTopFields, ResourceOptsMap), + ResourceOptsMap2 = emqx_connector_schema:project_to_connector_resource_opts(ResourceOptsMap), ConnectorConfigMap2 = maps:put(<<"resource_opts">>, ResourceOptsMap2, ConnectorConfigMap), IngressMap0 = maps:get(<<"ingress">>, Config, #{}), EgressMap = maps:get(<<"egress">>, Config, #{}), @@ -190,7 +185,13 @@ connector_action_config_to_bridge_v1_config( Params = maps:get(<<"parameters">>, ActionConfig, #{}), ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), - ResourceOpts = maps:merge(ResourceOptsConnector, ResourceOptsAction), + ResourceOpts0 = maps:merge(ResourceOptsConnector, ResourceOptsAction), + V1ResourceOptsFields = + lists:map( + fun({Field, _}) -> atom_to_binary(Field) end, + emqx_bridge_mqtt_schema:fields("creation_opts") + ), + ResourceOpts = maps:with(V1ResourceOptsFields, ResourceOpts0), %% Check the direction of the action Direction = case maps:get(<<"remote">>, Params) of @@ -212,11 +213,15 @@ connector_action_config_to_bridge_v1_config( #{<<"egress">> => Parms3}; {<<"publisher">>, LocalT} -> #{ - <<"egress">> => Parms3, - <<"local">> => - #{ - <<"topic">> => LocalT - } + <<"egress">> => + maps:merge( + Parms3, #{ + <<"local">> => + #{ + <<"topic">> => LocalT + } + } + ) }; {<<"subscriber">>, _} -> #{<<"ingress">> => Parms3} From 14b99737e9fbe63c122fb908b9da0a0d15bfc00f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 17:59:02 -0300 Subject: [PATCH 31/62] fix(mqtt_bridge): add missing fields to POST api spec; fix test --- .../test/emqx_bridge_confluent_tests.erl | 2 +- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl | 2 +- apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl index a7efebf89..2ea7fed22 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl @@ -72,7 +72,7 @@ parse(Hocon) -> Conf. check(SchemaMod, Conf) when is_map(Conf) -> - hocon_tconf:check_plain(SchemaMod, Conf). + hocon_tconf:check_plain(SchemaMod, Conf, #{required => false}). check_action(Conf) when is_map(Conf) -> check(emqx_bridge_v2_schema, Conf). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 2aba6e8ea..67dc52911 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -91,7 +91,7 @@ fields("get_connector") -> fields("get_bridge_v2") -> fields("mqtt_publisher_action"); fields("post_bridge_v2") -> - fields("mqtt_publisher_action"); + fields("mqtt_publisher_action") ++ emqx_bridge_schema:type_and_name_fields(mqtt); fields("put_bridge_v2") -> fields("mqtt_publisher_action"); fields(What) -> diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 0cefd6af4..a74e80fa1 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -166,11 +166,6 @@ common_init(Config0) -> #{work_dir => emqx_cth_suite:work_dir(Config0)} ), {ok, _Api} = emqx_common_test_http:create_default_app(), - - %% ok = emqx_common_test_helpers:start_apps([emqx, emqx_postgresql, emqx_conf, emqx_bridge]), - %% _ = emqx_bridge_enterprise:module_info(), - %% emqx_mgmt_api_test_util:init_suite(), - % Connect to pgsql directly and create the table connect_and_create_table(Config0), {Name, PGConf} = pgsql_config(BridgeType, Config0), From cc34660ab99590f8049eb91b41a65a28759b8ecc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:04:39 -0300 Subject: [PATCH 32/62] fix(actions): use backward-compatible ids --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 66d4dc674..ca421adea 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -939,7 +939,12 @@ id_with_root_name(RootName, BridgeType, BridgeName) -> ) end. -id_with_root_name(RootName, BridgeType, BridgeName, ConnectorName) -> +id_with_root_name(RootName0, BridgeType, BridgeName, ConnectorName) -> + RootName = + case bin(RootName0) of + <<"actions">> -> <<"action">>; + <<"sources">> -> <<"source">> + end, ConnectorType = bin(connector_type(BridgeType)), << (bin(RootName))/binary, From 697c8f5ee11072cc5669ba08aedbf83133289c2e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:34:10 -0300 Subject: [PATCH 33/62] test: fix broken tests --- apps/emqx_bridge/src/emqx_bridge.erl | 14 ++++++++------ apps/emqx_bridge/src/emqx_bridge_v2.erl | 1 + apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl | 17 ++++------------- .../emqx_bridge/test/emqx_bridge_v2_testlib.erl | 10 ++++++++-- .../test/emqx_bridge_http_SUITE.erl | 12 +++++++++--- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index c7d9a2d27..7df55c81c 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -347,14 +347,16 @@ lookup(Type, Name, RawConf) -> }} end. -get_metrics(Type, Name) -> - case emqx_bridge_v2:is_bridge_v2_type(Type) of +get_metrics(ActionType, Name) -> + case emqx_bridge_v2:is_bridge_v2_type(ActionType) of true -> - case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of + case emqx_bridge_v2:bridge_v1_is_valid(ActionType, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), + BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(ActionType), try - ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one(Type, Name), + ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( + ActionType, Name + ), emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) catch error:Reason -> @@ -364,7 +366,7 @@ get_metrics(Type, Name) -> {error, not_bridge_v1_compatible} end; false -> - emqx_resource:get_metrics(emqx_bridge_resource:resource_id(Type, Name)) + emqx_resource:get_metrics(emqx_bridge_resource:resource_id(ActionType, Name)) end. maybe_upgrade(mqtt, Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index ca421adea..38eb56e0f 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -46,6 +46,7 @@ %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies remove/2, + remove/3, %% The following is the remove function that is called by the HTTP API %% It also checks for rule action dependencies and optionally removes %% them diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 82efc77d2..1314fef48 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -188,18 +188,7 @@ end_per_testcase(_, Config) -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_bridge:remove(Type, Name) @@ -1026,9 +1015,11 @@ t_with_redact_update(Config) -> BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), + %% bridge is migrated after creation + ConfigRootKey = connectors, ?assertEqual( Password, - get_raw_config([bridges, Type, Name, password], Config) + get_raw_config([ConfigRootKey, Type, Name, password], Config) ), %% probe with new password; should not be considered redacted diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index eb8a9a5f8..5b821fea4 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -96,9 +96,15 @@ delete_all_bridges_and_connectors() -> delete_all_bridges() -> lists:foreach( fun(#{name := Name, type := Type}) -> - emqx_bridge_v2:remove(Type, Name) + emqx_bridge_v2:remove(actions, Type, Name) end, - emqx_bridge_v2:list() + emqx_bridge_v2:list(actions) + ), + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge_v2:remove(sources, Type, Name) + end, + emqx_bridge_v2:list(sources) ). delete_all_connectors() -> diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 3b7303300..f21e879b8 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -244,6 +244,12 @@ parse_http_request_assertive(ReqStr0) -> %% Helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +get_metrics(Name) -> + %% Note: `emqx_bridge:get_metrics/2' is currently *only* called in prod by + %% `emqx_bridge_api:lookup_from_local_node' with an action (not v1 bridge) type. + Type = <<"http">>, + emqx_bridge:get_metrics(Type, Name). + bridge_async_config(#{port := Port} = Config) -> Type = maps:get(type, Config, ?BRIDGE_TYPE), Name = maps:get(name, Config, ?BRIDGE_NAME), @@ -570,7 +576,7 @@ t_path_not_found(Config) -> success := 0 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -611,7 +617,7 @@ t_too_many_requests(Config) -> success := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ok @@ -654,7 +660,7 @@ t_rule_action_expired(Config) -> dropped := 1 } }, - emqx_bridge:get_metrics(?BRIDGE_TYPE, ?BRIDGE_NAME) + get_metrics(?BRIDGE_NAME) ) ), ?retry( From ab1b0dda677d09e169e2bf9a7ca7b623a573009a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 18:35:42 -0300 Subject: [PATCH 34/62] refactor: fix typo --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 38eb56e0f..c46013c0c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1345,13 +1345,13 @@ get_conf_root_key_if_only_one(BridgeType, BridgeName) -> LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, BridgeType, BridgeName), case {LookUpConfActions, LookUpConfSources} of {{error, bridge_not_found}, {error, bridge_not_found}} -> - error({action_or_soruces_not_found, BridgeType, BridgeName}); + error({action_or_source_not_found, BridgeType, BridgeName}); {{error, bridge_not_found}, _Conf} -> ?ROOT_KEY_SOURCES; {_Conf, {error, bridge_not_found}} -> ?ROOT_KEY_ACTIONS; {_Conf1, _Conf2} -> - error({name_clash_action_soruces, BridgeType, BridgeName}) + error({name_clash_action_source, BridgeType, BridgeName}) end. lookup_conf(RootName, Type, Name) -> From 3597ee7c939dbe1db421cfb8f66f25a00425ab37 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 19:00:47 -0300 Subject: [PATCH 35/62] fix(mqtt_action): fix resource_opts schema --- .../src/emqx_bridge_mqtt_pubsub_action_info.erl | 4 ++-- .../emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index de39fc9b4..8918a60be 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -88,7 +88,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_publisher_action"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), @@ -113,7 +113,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("resource_opts"), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 67dc52911..2cc13daaf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -80,11 +80,11 @@ fields(ingress_parameters) -> Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), Fields1 = proplists:delete(pool_size, Fields0), Fields1; -fields("resource_opts") -> +fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_resource_schema:fields("creation_opts") + emqx_bridge_v2_schema:resource_opts_fields() ); fields("get_connector") -> emqx_bridge_mqtt_connector_schema:fields("config_connector"); @@ -105,7 +105,7 @@ fields(What) -> desc("config") -> ?DESC("desc_config"); -desc("resource_opts") -> +desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; From 1ad3100cad9a6398aa549212944c308838aab3e9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 20:02:04 -0300 Subject: [PATCH 36/62] chore: add i18n --- .../src/emqx_bridge_mqtt_connector_schema.erl | 4 ++++ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 8 +++++++ .../emqx_bridge_mqtt_connector_schema.hocon | 5 +++++ rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon | 22 +++++++++++++++++++ rel/i18n/emqx_bridge_v2_schema.hocon | 10 +++++++-- 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index ba4373313..83be577f4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -338,6 +338,8 @@ egress_pool_size(Prop) -> desc("server_configs") -> ?DESC("server_configs"); +desc("config_connector") -> + ?DESC("config_connector"); desc("ingress") -> ?DESC("ingress_desc"); desc("ingress_remote") -> @@ -350,6 +352,8 @@ desc("egress_remote") -> ?DESC("egress_remote"); desc("egress_local") -> ?DESC("egress_local"); +desc(resource_opts) -> + ?DESC(emqx_resource_schema, <<"resource_opts">>); desc(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 2cc13daaf..6d075334a 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -107,6 +107,10 @@ desc("config") -> ?DESC("desc_config"); desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); +desc(action_parameters) -> + ?DESC(action_parameters); +desc(ingress_parameters) -> + ?DESC(ingress_parameters); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; desc("config_connector") -> @@ -115,6 +119,10 @@ desc("http_action") -> ?DESC("desc_config"); desc("parameters_opts") -> ?DESC("config_parameters_opts"); +desc("mqtt_publisher_action") -> + ?DESC("mqtt_publisher_action"); +desc("mqtt_subscriber_source") -> + ?DESC("mqtt_subscriber_source"); desc(_) -> undefined. diff --git a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon index 7c7bf68c9..25bb8aad2 100644 --- a/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_connector_schema.hocon @@ -194,4 +194,9 @@ username.desc: username.label: """Username""" +config_connector.desc: +"""Configurations for an MQTT connector.""" +config_connector.label: +"""MQTT connector""" + } diff --git a/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon new file mode 100644 index 000000000..637189781 --- /dev/null +++ b/rel/i18n/emqx_bridge_mqtt_pubsub_schema.hocon @@ -0,0 +1,22 @@ +emqx_bridge_mqtt_pubsub_schema { + action_parameters.desc: + """Action specific configs.""" + action_parameters.label: + """Action""" + + ingress_parameters.desc: + """Source specific configs.""" + ingress_parameters.label: + """Source""" + + mqtt_publisher_action.desc: + """Action configs.""" + mqtt_publisher_action.label: + """Action""" + + mqtt_subscriber_source.desc: + """Source configs.""" + mqtt_subscriber_source.label: + """Source""" + +} diff --git a/rel/i18n/emqx_bridge_v2_schema.hocon b/rel/i18n/emqx_bridge_v2_schema.hocon index 69f8a9109..3a4bf6140 100644 --- a/rel/i18n/emqx_bridge_v2_schema.hocon +++ b/rel/i18n/emqx_bridge_v2_schema.hocon @@ -1,10 +1,16 @@ emqx_bridge_v2_schema { desc_bridges_v2.desc: -"""Configuration for bridges.""" +"""Configuration for actions.""" desc_bridges_v2.label: -"""Bridge Configuration""" +"""Action Configuration""" + +desc_sources.desc: +"""Configuration for sources.""" + +desc_sources.label: +"""Source Configuration""" mqtt_topic.desc: """MQTT topic or topic filter as data source (action input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in the remote system.""" From 7fc069da46302afb8c240ef10c8cf0810777c63a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 19 Dec 2023 20:02:54 -0300 Subject: [PATCH 37/62] test: fix another broken test --- apps/emqx_bridge/test/emqx_bridge_SUITE.erl | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index 30107d0ce..19ae516ec 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -32,7 +32,9 @@ init_per_suite(Config) -> emqx_conf, emqx_connector, emqx_bridge_http, - emqx_bridge + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine ], #{work_dir => ?config(priv_dir, Config)} ), @@ -154,14 +156,18 @@ setup_fake_telemetry_data() -> ok. t_update_ssl_conf(Config) -> - Path = proplists:get_value(config_path, Config), - CertDir = filename:join([emqx:mutable_certs_dir() | Path]), + [_Root, Type, Name] = proplists:get_value(config_path, Config), + CertDir = filename:join([emqx:mutable_certs_dir(), connectors, Type, Name]), EnableSSLConf = #{ <<"bridge_mode">> => false, <<"clean_start">> => true, <<"keepalive">> => <<"60s">>, <<"proto_ver">> => <<"v4">>, <<"server">> => <<"127.0.0.1:1883">>, + <<"egress">> => #{ + <<"local">> => #{<<"topic">> => <<"t">>}, + <<"remote">> => #{<<"topic">> => <<"remote/t">>} + }, <<"ssl">> => #{ <<"cacertfile">> => cert_file("cafile"), @@ -171,10 +177,15 @@ t_update_ssl_conf(Config) -> <<"verify">> => <<"verify_peer">> } }, - {ok, _} = emqx:update_config(Path, EnableSSLConf), + CreateCfg = [ + {bridge_name, Name}, + {bridge_type, Type}, + {bridge_config, #{}} + ], + {ok, _} = emqx_bridge_testlib:create_bridge_api(CreateCfg, EnableSSLConf), ?assertMatch({ok, [_, _, _]}, file:list_dir(CertDir)), NoSSLConf = EnableSSLConf#{<<"ssl">> := #{<<"enable">> => false}}, - {ok, _} = emqx:update_config(Path, NoSSLConf), + {ok, _} = emqx_bridge_testlib:update_bridge_api(CreateCfg, NoSSLConf), {ok, _} = emqx_tls_certfile_gc:force(), ?assertMatch({error, enoent}, file:list_dir(CertDir)), ok. From 862283ff7c7d5d12ae236ce256729e440dce0b49 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 20 Dec 2023 13:10:50 -0300 Subject: [PATCH 38/62] test: fix expected connector name after name convention generation changed --- .../test/emqx_bridge_gcp_pubsub_producer_SUITE.erl | 8 ++++++-- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl index e030bbc74..92e4e09a6 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl @@ -159,7 +159,7 @@ generate_config(Config0) -> } = gcp_pubsub_config(Config0), %% FIXME %% `emqx_bridge_resource:resource_id' requires an existing connector in the config..... - ConnectorName = <<"connector_", ActionName/binary>>, + ConnectorName = ActionName, ConnectorResourceId = <<"connector:", ?CONNECTOR_TYPE_BIN/binary, ":", ConnectorName/binary>>, ActionResourceId = emqx_bridge_v2:id(?ACTION_TYPE_BIN, ActionName, ConnectorName), BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_V1_TYPE_BIN, ActionName), @@ -1228,7 +1228,11 @@ do_econnrefused_or_timeout_test(Config, Error) -> %% _Msg = "The connection was lost." ok; Trace0 -> - error({unexpected_trace, Trace0}) + error( + {unexpected_trace, Trace0, #{ + expected_connector_id => ConnectorResourceId + }} + ) end; timeout -> ?assertMatch( diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index bb3ca87b7..6e520ba58 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -412,7 +412,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"webhook_name">> => #{ - <<"connector">> => <<"connector_webhook_name">>, + <<"connector">> => <<"webhook_name">>, <<"description">> => <<>>, <<"enable">> => true, <<"parameters">> => @@ -440,7 +440,7 @@ t_create_webhook_v1_bridges_api(Config) -> #{ <<"http">> => #{ - <<"connector_webhook_name">> => + <<"webhook_name">> => #{ <<"connect_timeout">> => <<"15s">>, <<"description">> => <<>>, From 12dc9fbeb9532f6e4fd0dd35ee5f27e8e02862ed Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 20 Dec 2023 15:45:51 -0300 Subject: [PATCH 39/62] test(mqtt_bridge): fix assertion --- .../test/emqx_bridge_mqtt_SUITE.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index bd3fb68de..3c50e16d8 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -410,13 +410,16 @@ t_mqtt_egress_bridge_warns_clean_start(_) -> ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []) + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []), + + ok end, - ?wait_async_action( - Action(), - #{?snk_kind := mqtt_clean_start_egress_action_warning}, - 10000 - ), + {ok, {ok, _}} = + ?wait_async_action( + Action(), + #{?snk_kind := mqtt_clean_start_egress_action_warning}, + 10000 + ), ok. t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> From 6511693b2ee139216a93cf81f8a719602d691f40 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 21 Dec 2023 14:14:41 -0300 Subject: [PATCH 40/62] refactor(action_api): prepare for `/sources` HTTP API --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_bridge/src/emqx_bridge_api.erl | 24 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 42 +++- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 216 ++++++++++++------ .../src/proto/emqx_bridge_proto_v6.erl | 196 ++++++++++++++++ .../test/emqx_bridge_api_SUITE.erl | 7 +- apps/emqx_bridge/test/emqx_bridge_testlib.erl | 16 +- .../test/emqx_bridge_v2_testlib.erl | 18 +- 8 files changed, 404 insertions(+), 116 deletions(-) create mode 100644 apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 9721a7f2f..9bd824242 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -8,6 +8,7 @@ {emqx_bridge,3}. {emqx_bridge,4}. {emqx_bridge,5}. +{emqx_bridge,6}. {emqx_broker,1}. {emqx_cm,1}. {emqx_cm,2}. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 3168ae590..8f36fd700 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,6 +49,11 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). +%% only for testting/mocking +-export([supported_versions/1]). + +-define(BPAPI_NAME, emqx_bridge). + -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). @@ -1102,18 +1107,18 @@ maybe_try_restart(_, _, _) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> case lists:member(Node, mria:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. do_bpapi_call_vsn(SupportedVersion, Call, Args) -> - case lists:member(SupportedVersion, supported_versions(Call)) of + case lists:member(SupportedVersion, ?MODULE:supported_versions(Call)) of true -> apply(emqx_bridge_proto_v4, Call, Args); false -> @@ -1125,10 +1130,15 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2, 3, 4, 5]; -supported_versions(start_bridges_to_all_nodes) -> [2, 3, 4, 5]; -supported_versions(get_metrics_from_all_nodes) -> [4, 5]; -supported_versions(_Call) -> [1, 2, 3, 4, 5]. +supported_versions(start_bridge_to_node) -> bpapi_version_range(2, latest); +supported_versions(start_bridges_to_all_nodes) -> bpapi_version_range(2, latest); +supported_versions(get_metrics_from_all_nodes) -> bpapi_version_range(4, latest); +supported_versions(_Call) -> bpapi_version_range(1, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). redact(Term) -> emqx_utils:redact(Term). diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index c46013c0c..fb8ae2e2e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -43,6 +43,7 @@ lookup/2, lookup/3, create/3, + create/4, %% The remove/2 function is only for internal use as it may create %% rules with broken dependencies remove/2, @@ -50,7 +51,8 @@ %% The following is the remove function that is called by the HTTP API %% It also checks for rule action dependencies and optionally removes %% them - check_deps_and_remove/3 + check_deps_and_remove/3, + check_deps_and_remove/4 ]). %% Operations @@ -62,9 +64,11 @@ send_message/4, query/4, start/2, + start/3, reset_metrics/2, reset_metrics/3, create_dry_run/2, + create_dry_run/3, get_metrics/2, get_metrics/3 ]). @@ -150,6 +154,10 @@ -type bridge_v2_type() :: binary() | atom() | [byte()]. -type bridge_v2_name() :: binary() | atom() | [byte()]. +-type root_cfg_key() :: ?ROOT_KEY_ACTIONS | ?ROOT_KEY_SOURCES. + +-export_type([root_cfg_key/0]). + %%==================================================================== %%==================================================================== @@ -212,7 +220,7 @@ unload_bridges(ConfRooKey) -> lookup(Type, Name) -> lookup(?ROOT_KEY_ACTIONS, Type, Name). --spec lookup(sources | actions, bridge_v2_type(), bridge_v2_name()) -> +-spec lookup(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(ConfRootName, Type, Name) -> case emqx:get_raw_config([ConfRootName, Type, Name], not_found) of @@ -315,6 +323,11 @@ remove(ConfRootKey, BridgeType, BridgeName) -> -spec check_deps_and_remove(bridge_v2_type(), bridge_v2_name(), boolean()) -> ok | {error, any()}. check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> + check_deps_and_remove(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, AlsoDeleteActions). + +-spec check_deps_and_remove(root_cfg_key(), bridge_v2_type(), bridge_v2_name(), boolean()) -> + ok | {error, any()}. +check_deps_and_remove(ConfRooKey, BridgeType, BridgeName, AlsoDeleteActions) -> AlsoDelete = case AlsoDeleteActions of true -> [rule_actions]; @@ -328,7 +341,7 @@ check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) -> ) of ok -> - remove(BridgeType, BridgeName); + remove(ConfRooKey, BridgeType, BridgeName); {error, Reason} -> {error, Reason} end. @@ -539,11 +552,14 @@ disable_enable(ConfRootKey, Action, BridgeType, BridgeName) when ?ENABLE_OR_DISA %% is something else than connected after starting the connector or if an %% error occurred when the connector was started. -spec start(term(), term()) -> ok | {error, Reason :: term()}. -start(BridgeV2Type, Name) -> +start(ActionOrSourceType, Name) -> + start(?ROOT_KEY_ACTIONS, ActionOrSourceType, Name). + +-spec start(root_cfg_key(), term(), term()) -> ok | {error, Reason :: term()}. +start(ConfRootKey, BridgeV2Type, Name) -> ConnectorOpFun = fun(ConnectorType, ConnectorName) -> emqx_connector_resource:start(ConnectorType, ConnectorName) end, - ConfRootKey = ?ROOT_KEY_ACTIONS, connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, true). connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHealthCheck) -> @@ -694,10 +710,15 @@ health_check(ConfRootKey, BridgeType, BridgeName) -> end. -spec create_dry_run(bridge_v2_type(), Config :: map()) -> ok | {error, term()}. -create_dry_run(Type, Conf0) -> +create_dry_run(Type, Conf) -> + create_dry_run(?ROOT_KEY_ACTIONS, Type, Conf). + +-spec create_dry_run(root_cfg_key(), bridge_v2_type(), Config :: map()) -> ok | {error, term()}. +create_dry_run(ConfRootKey, Type, Conf0) -> Conf1 = maps:without([<<"name">>], Conf0), TypeBin = bin(Type), - RawConf = #{<<"actions">> => #{TypeBin => #{<<"temp_name">> => Conf1}}}, + ConfRootKeyBin = bin(ConfRootKey), + RawConf = #{ConfRootKeyBin => #{TypeBin => #{<<"temp_name">> => Conf1}}}, %% Check config try _ = @@ -722,6 +743,9 @@ create_dry_run(Type, Conf0) -> end. create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> + create_dry_run_helper(?ROOT_KEY_ACTIONS, BridgeType, ConnectorRawConf, BridgeV2RawConf). + +create_dry_run_helper(ConfRootKey, BridgeType, ConnectorRawConf, BridgeV2RawConf) -> BridgeName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), ConnectorType = connector_type(BridgeType), OnReadyCallback = @@ -730,7 +754,7 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> ChannelTestId = id(BridgeType, BridgeName, ConnectorName), Conf = emqx_utils_maps:unsafe_atom_key_map(BridgeV2RawConf), AugmentedConf = augment_channel_config( - ?ROOT_KEY_ACTIONS, + ConfRootKey, BridgeType, BridgeName, Conf @@ -756,6 +780,8 @@ create_dry_run_helper(BridgeType, ConnectorRawConf, BridgeV2RawConf) -> get_metrics(Type, Name) -> get_metrics(?ROOT_KEY_ACTIONS, Type, Name). +-spec get_metrics(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> + emqx_metrics_worker:metrics(). get_metrics(ConfRootKey, Type, Name) -> emqx_resource:get_metrics(id_with_root_name(ConfRootKey, Type, Name)). diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 254390a36..3f0d18fae 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -26,6 +26,9 @@ -import(hoconsc, [mk/2, array/1, enum/1]). -import(emqx_utils, [redact/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% Swagger specs from hocon schema -export([ api_spec/0, @@ -48,7 +51,14 @@ ]). %% BpAPI / RPC Targets --export([lookup_from_local_node/2, get_metrics_from_local_node/2]). +-export([ + lookup_from_local_node/2, + get_metrics_from_local_node/2, + lookup_from_local_node_v6/3, + get_metrics_from_local_node_v6/3 +]). + +-define(BPAPI_NAME, emqx_bridge). -define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME), ?NOT_FOUND( @@ -393,16 +403,51 @@ schema("/action_types") -> }. '/actions'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, _} -> - ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); - {error, not_found} -> - Conf = filter_out_request_body(Conf0), - create_bridge(BridgeType, BridgeName, Conf) - end; + handle_create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, Conf0); '/actions'(get, _Params) -> - Nodes = mria:running_nodes(), - NodeReplies = emqx_bridge_proto_v5:v2_list_bridges_on_nodes(Nodes), + handle_list(?ROOT_KEY_ACTIONS). + +'/actions/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, 200)); +'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_ACTIONS, Id, Conf0); +'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_ACTIONS, Id, Qs). + +'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_ACTIONS, BridgeType, BridgeName)). + +'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_ACTIONS, Id). + +'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_ACTIONS, Id, Enable). + +'/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_ACTIONS, Id, Op). + +'/nodes/:node/actions/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_ACTIONS, Node, Id, Op). + +'/actions_probe'(post, Request) -> + handle_probe(?ROOT_KEY_ACTIONS, Request). + +'/action_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:types()). + +%%------------------------------------------------------------------------------ +%% Handlers +%%------------------------------------------------------------------------------ + +handle_list(ConfRootKey) -> + Nodes = emqx:running_nodes(), + NodeReplies = emqx_bridge_proto_v6:v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey), case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ @@ -414,34 +459,44 @@ schema("/action_types") -> ?INTERNAL_ERROR(Reason) end. -'/actions/:id'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); -'/actions/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> +handle_create(ConfRootKey, Type, Name, Conf0) -> + case emqx_bridge_v2:lookup(ConfRootKey, Type, Name) of + {ok, _} -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>); + {error, not_found} -> + Conf = filter_out_request_body(Conf0), + create_bridge(ConfRootKey, Type, Name, Conf) + end. + +handle_update(ConfRootKey, Id, Conf0) -> Conf1 = filter_out_request_body(Conf0), ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), Conf = emqx_utils:deobfuscate(Conf1, RawConf), - update_bridge(BridgeType, BridgeName, Conf); + update_bridge(ConfRootKey, BridgeType, BridgeName, Conf); {error, not_found} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end - ); -'/actions/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + ). + +handle_delete(ConfRootKey, Id, QueryStringOpts) -> ?TRY_PARSE_ID( Id, case emqx_bridge_v2:lookup(BridgeType, BridgeName) of {ok, _} -> AlsoDeleteActions = - case maps:get(<<"also_delete_dep_actions">>, Qs, <<"false">>) of + case maps:get(<<"also_delete_dep_actions">>, QueryStringOpts, <<"false">>) of <<"true">> -> true; true -> true; _ -> false end, case - emqx_bridge_v2:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActions) + emqx_bridge_v2:check_deps_and_remove( + ConfRootKey, BridgeType, BridgeName, AlsoDeleteActions + ) of ok -> ?NO_CONTENT; @@ -465,23 +520,22 @@ schema("/action_types") -> end ). -'/actions/:id/metrics'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). - -'/actions/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> +handle_reset_metrics(ConfRootKey, Id) -> ?TRY_PARSE_ID( Id, begin ActionType = emqx_bridge_v2:bridge_v2_type_to_connector_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(ActionType, BridgeName), + ok = emqx_bridge_v2:reset_metrics(ConfRootKey, ActionType, BridgeName), ?NO_CONTENT end ). -'/actions/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> +handle_disable_enable(ConfRootKey, Id, Enable) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:disable_enable(enable_func(Enable), BridgeType, BridgeName) of + case + emqx_bridge_v2:disable_enable(ConfRootKey, enable_func(Enable), BridgeType, BridgeName) + of {ok, _} -> ?NO_CONTENT; {error, {pre_config_update, _, bridge_not_found}} -> @@ -495,41 +549,37 @@ schema("/action_types") -> end ). -'/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op} -}) -> +handle_operation(ConfRootKey, Id, Op) -> ?TRY_PARSE_ID( Id, begin OperFunc = operation_func(all, Op), - Nodes = mria:running_nodes(), - call_operation_if_enabled(all, OperFunc, [Nodes, BridgeType, BridgeName]) + Nodes = emqx:running_nodes(), + call_operation_if_enabled(all, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) end ). -'/nodes/:node/actions/:id/:operation'(post, #{ - bindings := - #{id := Id, operation := Op, node := Node} -}) -> +handle_node_operation(ConfRootKey, Node, Id, Op) -> ?TRY_PARSE_ID( Id, case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> OperFunc = operation_func(TargetNode, Op), - call_operation_if_enabled(TargetNode, OperFunc, [TargetNode, BridgeType, BridgeName]); + call_operation_if_enabled(TargetNode, OperFunc, [ + TargetNode, ConfRootKey, BridgeType, BridgeName + ]); {error, _} -> ?NOT_FOUND(<<"Invalid node name: ", Node/binary>>) end ). -'/actions_probe'(post, Request) -> +handle_probe(ConfRootKey, Request) -> RequestMeta = #{module => ?MODULE, method => post, path => "/actions_probe"}, case emqx_dashboard_swagger:filter_check_request_and_translate_body(Request, RequestMeta) of - {ok, #{body := #{<<"type">> := ConnType} = Params}} -> + {ok, #{body := #{<<"type">> := Type} = Params}} -> Params1 = maybe_deobfuscate_bridge_probe(Params), Params2 = maps:remove(<<"type">>, Params1), - case emqx_bridge_v2:create_dry_run(ConnType, Params2) of + case emqx_bridge_v2:create_dry_run(ConfRootKey, Type, Params2) of ok -> ?NO_CONTENT; {error, #{kind := validation_error} = Reason0} -> @@ -548,9 +598,7 @@ schema("/action_types") -> redact(BadRequest) end. -'/action_types'(get, _Request) -> - ?OK(emqx_bridge_v2_schema:types()). - +%%% API helpers maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) -> case emqx_bridge_v2:lookup(ActionType, BridgeName) of {ok, #{raw_config := RawConf}} -> @@ -564,7 +612,6 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeN maybe_deobfuscate_bridge_probe(Params) -> Params. -%%% API helpers is_ok(ok) -> ok; is_ok(OkResult = {ok, _}) -> @@ -587,9 +634,16 @@ is_ok(ResL) -> end. %% bridge helpers -lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> - Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v5:v2_lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of +-spec lookup_from_all_nodes(emqx_bridge_v2:root_cfg_key(), _, _, _) -> _. +lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, SuccCode) -> + Nodes = emqx:running_nodes(), + case + is_ok( + emqx_bridge_proto_v6:v2_lookup_from_all_nodes_v6( + Nodes, ConfRootKey, BridgeType, BridgeName + ) + ) + of {ok, [{ok, _} | _] = Results} -> {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> @@ -598,10 +652,10 @@ lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> ?INTERNAL_ERROR(Reason) end. -get_metrics_from_all_nodes(ActionType, ActionName) -> +get_metrics_from_all_nodes(ConfRootKey, Type, Name) -> Nodes = emqx:running_nodes(), Result = maybe_unwrap( - emqx_bridge_proto_v5:v2_get_metrics_from_all_nodes(Nodes, ActionType, ActionName) + emqx_bridge_proto_v6:v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, Type, Name) ), case Result of Metrics when is_list(Metrics) -> @@ -610,22 +664,25 @@ get_metrics_from_all_nodes(ActionType, ActionName) -> ?INTERNAL_ERROR(Reason) end. -operation_func(all, start) -> v2_start_bridge_to_all_nodes; -operation_func(_Node, start) -> v2_start_bridge_to_node. +operation_func(all, start) -> v2_start_bridge_to_all_nodes_v6; +operation_func(_Node, start) -> v2_start_bridge_to_node_v6; +operation_func(all, lookup) -> v2_lookup_from_all_nodes_v6; +operation_func(all, list) -> v2_list_bridges_on_nodes_v6; +operation_func(all, get_metrics) -> v2_get_metrics_from_all_nodes_v6. -call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) -> - try is_enabled_bridge(BridgeType, BridgeName) of +call_operation_if_enabled(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) -> + try is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) of false -> ?BRIDGE_NOT_ENABLED; true -> - call_operation(NodeOrAll, OperFunc, [Nodes, BridgeType, BridgeName]) + call_operation(NodeOrAll, OperFunc, [Nodes, ConfRootKey, BridgeType, BridgeName]) catch throw:not_found -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName) end. -is_enabled_bridge(BridgeType, BridgeName) -> - try emqx_bridge_v2:lookup(BridgeType, binary_to_existing_atom(BridgeName)) of +is_enabled_bridge(ConfRootKey, BridgeType, BridgeName) -> + try emqx_bridge_v2:lookup(ConfRootKey, BridgeType, binary_to_existing_atom(BridgeName)) of {ok, #{raw_config := ConfMap}} -> maps:get(<<"enable">>, ConfMap, false); {error, not_found} -> @@ -637,7 +694,7 @@ is_enabled_bridge(BridgeType, BridgeName) -> throw(not_found) end. -call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> +call_operation(NodeOrAll, OperFunc, Args = [_Nodes, _ConfRootKey, BridgeType, BridgeName]) -> case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok -> ?NO_CONTENT; @@ -668,12 +725,12 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> do_bpapi_call(all, Call, Args) -> maybe_unwrap( - do_bpapi_call_vsn(emqx_bpapi:supported_version(emqx_bridge), Call, Args) + do_bpapi_call_vsn(emqx_bpapi:supported_version(?BPAPI_NAME), Call, Args) ); do_bpapi_call(Node, Call, Args) -> - case lists:member(Node, mria:running_nodes()) of + case lists:member(Node, emqx:running_nodes()) of true -> - do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, emqx_bridge), Call, Args); + do_bpapi_call_vsn(emqx_bpapi:supported_version(Node, ?BPAPI_NAME), Call, Args); false -> {error, {node_not_found, Node}} end. @@ -681,7 +738,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(Version, Call, Args) -> case is_supported_version(Version, Call) of true -> - apply(emqx_bridge_proto_v5, Call, Args); + apply(emqx_bridge_proto_v6, Call, Args); false -> {error, not_implemented} end. @@ -689,7 +746,12 @@ do_bpapi_call_vsn(Version, Call, Args) -> is_supported_version(Version, Call) -> lists:member(Version, supported_versions(Call)). -supported_versions(_Call) -> [5]. +supported_versions(_Call) -> bpapi_version_range(6, latest). + +%% [From, To] (inclusive on both ends) +bpapi_version_range(From, latest) -> + ThisNodeVsn = emqx_bpapi:supported_version(node(), ?BPAPI_NAME), + lists:seq(From, ThisNodeVsn). maybe_unwrap({error, not_implemented}) -> {error, not_implemented}; @@ -767,10 +829,22 @@ lookup_from_local_node(BridgeType, BridgeName) -> Error -> Error end. +%% RPC Target +-spec lookup_from_local_node_v6(emqx_bridge_v2:root_cfg_key(), _, _) -> _. +lookup_from_local_node_v6(ConfRootKey, BridgeType, BridgeName) -> + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of + {ok, Res} -> {ok, format_resource(Res, node())}; + Error -> Error + end. + %% RPC Target get_metrics_from_local_node(ActionType, ActionName) -> format_metrics(emqx_bridge_v2:get_metrics(ActionType, ActionName)). +%% RPC Target +get_metrics_from_local_node_v6(ConfRootKey, Type, Name) -> + format_metrics(emqx_bridge_v2:get_metrics(ConfRootKey, Type, Name)). + %% resource format_resource( #{ @@ -938,13 +1012,13 @@ format_resource_data(error, Error, Result) -> format_resource_data(K, V, Result) -> Result#{K => V}. -create_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 201). +create_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 201). -update_bridge(BridgeType, BridgeName, Conf) -> - create_or_update_bridge(BridgeType, BridgeName, Conf, 200). +update_bridge(ConfRootKey, BridgeType, BridgeName, Conf) -> + create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, 200). -create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> +create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> Check = try is_binary(BridgeType) andalso emqx_resource:validate_type(BridgeType), @@ -955,15 +1029,15 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> end, case Check of ok -> - do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode); + do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode); BadRequest -> BadRequest end. -do_create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> - case emqx_bridge_v2:create(BridgeType, BridgeName, Conf) of +do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCode) -> + case emqx_bridge_v2:create(ConfRootKey, BridgeType, BridgeName, Conf) of {ok, _} -> - lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode); + lookup_from_all_nodes(ConfRootKey, BridgeType, BridgeName, HttpStatusCode); {error, {PreOrPostConfigUpdate, _HandlerMod, Reason}} when PreOrPostConfigUpdate =:= pre_config_update; PreOrPostConfigUpdate =:= post_config_update diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl new file mode 100644 index 000000000..d6fe68466 --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl @@ -0,0 +1,196 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v6). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + get_metrics_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3, + + %% introduced in v6 + v2_lookup_from_all_nodes_v6/4, + v2_list_bridges_on_nodes_v6/2, + v2_get_metrics_from_all_nodes_v6/4, + v2_start_bridge_to_node_v6/4, + v2_start_bridge_to_all_nodes_v6/4 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.5.0". + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(ok). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(term()). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()). +get_metrics_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + get_metrics_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +%%-------------------------------------------------------------------------------- +%% introduced in v6 +%%-------------------------------------------------------------------------------- + +%% V2 Calls +-spec v2_lookup_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_lookup_from_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + lookup_from_local_node_v6, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_list_bridges_on_nodes_v6([node()], emqx_bridge_v2:root_cfg_key()) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +v2_list_bridges_on_nodes_v6(Nodes, ConfRootKey) -> + erpc:multicall(Nodes, emqx_bridge_v2, list, [ConfRootKey], ?TIMEOUT). + +-spec v2_get_metrics_from_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(term()). +v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, ActionType, ActionName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2_api, + get_metrics_from_local_node_v6, + [ConfRootKey, ActionType, ActionName], + ?TIMEOUT + ). + +-spec v2_start_bridge_to_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> + emqx_rpc:erpc_multicall(ok). +v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec v2_start_bridge_to_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> + term(). +v2_start_bridge_to_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_v2, + start, + [ConfRootKey, BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 1314fef48..112e24e63 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -160,8 +160,9 @@ end_per_group(_, Config) -> init_per_testcase(t_broken_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), - meck:expect(emqx_bpapi, supported_version, 1, -1), meck:expect(emqx_bpapi, supported_version, 2, -1), + meck:new(emqx_bridge_api, [passthrough]), + meck:expect(emqx_bridge_api, supported_versions, 1, []), init_per_testcase(common, Config); init_per_testcase(t_old_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), @@ -173,10 +174,10 @@ init_per_testcase(_, Config) -> [{port, Port}, {sock, Sock}, {acceptor, Acceptor} | Config]. end_per_testcase(t_broken_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(t_old_bpapi_vsn, Config) -> - meck:unload([emqx_bpapi]), + meck:unload(), end_per_testcase(common, Config); end_per_testcase(_, Config) -> Sock = ?config(sock, Config), diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index df4560e6e..7ab8c68a6 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -196,20 +196,10 @@ delete_bridge_http_api_v1(Opts) -> op_bridge_api(Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, + Method = post, + Params = [], + Res = emqx_bridge_v2_testlib:request(Method, Path, Params), ct:pal("bridge op result: ~p", [Res]), Res. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 5b821fea4..f7dd74161 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -308,21 +308,11 @@ update_bridge_api(Config, Overrides) -> op_bridge_api(Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId, Op]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, "", Opts) of - {ok, {Status = {_, 204, _}, Headers, Body}} -> - {ok, {Status, Headers, Body}}; - {ok, {Status, Headers, Body}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - {error, {Status, Headers, Body}} -> - {error, {Status, Headers, emqx_utils_json:decode(Body, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge op result: ~p", [Res]), + Method = post, + Params = [], + Res = request(Method, Path, Params), + ct:pal("bridge op result:\n ~p", [Res]), Res. probe_bridge_api(Config) -> From e6ccfa5b397e275e6512422d273e07c18637b097 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Jan 2024 12:17:26 -0300 Subject: [PATCH 41/62] fix(mqtt_bridge): fixes after rebasing onto current `master` Rebased on top of 7f57ec47d5f634da67a63ede531b5e85f2a229b6 --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 7 +--- .../src/emqx_bridge_mqtt_connector_schema.erl | 40 ++++++------------- .../emqx_bridge_mqtt_pubsub_action_info.erl | 2 +- .../test/emqx_bridge_mqtt_SUITE.erl | 15 +++---- 5 files changed, 24 insertions(+), 42 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e6fe78ab8..0c00a0d59 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 6f9465cb0..dbdf68ef1 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -414,10 +414,8 @@ mk_client_opts( ssl_opts => maps:to_list(maps:remove(enable, Ssl)) }). -parse_id_to_name(<>) -> - Name; parse_id_to_name(Id) -> - {_Type, Name} = emqx_bridge_resource:parse_bridge_id(Id, #{atom_name => false}), + {_Type, Name} = emqx_connector_resource:parse_connector_id(Id, #{atom_name => false}), Name. mk_client_opt_password(Options = #{password := Secret}) -> @@ -447,7 +445,6 @@ connect(Options) -> }), Name = proplists:get_value(name, Options), WorkerId = proplists:get_value(ecpool_worker_id, Options), - WorkerId = proplists:get_value(ecpool_worker_id, Options), ClientOpts = proplists:get_value(client_opts, Options), case emqtt:start_link(mk_client_opts(Name, WorkerId, ClientOpts)) of {ok, Pid} -> @@ -475,7 +472,7 @@ mk_client_opts( }. mk_clientid(WorkerId, ClientId) -> - iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]). + emqx_bridge_mqtt_lib:bytes23([ClientId], WorkerId). mk_client_event_handler(Name, TopicToHandlerIndex) -> #{ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 83be577f4..edbb40cbb 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -38,6 +38,7 @@ -import(hoconsc, [mk/2, ref/2]). +-define(CONNECTOR_TYPE, mqtt). -define(MQTT_HOST_OPTS, #{default_port => 1883}). namespace() -> "connector_mqtt". @@ -66,28 +67,10 @@ fields("config") -> )} ]; fields("config_connector") -> - [ - {enable, - mk( - boolean(), - #{ - desc => <<"Enable or disable this connector">>, - default => true - } - )}, - {description, emqx_schema:description_schema()}, - {pool_size, fun egress_pool_size/1} - % {ingress, - % mk( - % hoconsc:array( - % hoconsc:ref(connector_ingress) - % ), - % #{ - % required => {false, recursively}, - % desc => ?DESC("ingress_desc") - % } - % )} - ] ++ emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ + emqx_connector_schema:common_fields() ++ fields("specific_connector_config"); +fields("specific_connector_config") -> + [{pool_size, fun egress_pool_size/1}] ++ + emqx_connector_schema:resource_opts_ref(?MODULE, resource_opts) ++ fields("server_configs"); fields(resource_opts) -> emqx_connector_schema:resource_opts_fields(); @@ -317,12 +300,13 @@ fields("egress_remote") -> } )} ]; -fields("get_connector") -> - fields("config_connector"); -fields("post_connector") -> - fields("config_connector"); -fields("put_connector") -> - fields("config_connector"); +fields(Field) when + Field == "get_connector"; + Field == "put_connector"; + Field == "post_connector" +-> + Fields = fields("specific_connector_config"), + emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, Fields); fields(What) -> error({emqx_bridge_mqtt_connector_schema, missing_field_handler, What}). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index 8918a60be..e4a4fcd19 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -94,7 +94,7 @@ bridge_v1_config_to_action_config_helper( ), LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), - EgressMap1 = maps:remove(<<"local">>, EgressMap0), + EgressMap1 = maps:without([<<"local">>, <<"pool_size">>], EgressMap0), %% Add parameters field (Egress map) to the action config ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), ConfigMap3 = diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 3c50e16d8..807fba3c9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -564,17 +564,17 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> t_egress_short_clientid(_Config) -> %% Name is short, expect the actual client ID in use is hashed from - %% E: - Name = "abc01234", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc01234">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = iolist_to_binary([BaseId, $:, "1"]), test_egress_clientid(Name, ExpectedClientId). t_egress_long_clientid(_Config) -> %% Expect the actual client ID in use is hashed from - %% E: - Name = "abc01234567890123456789", - BaseId = emqx_bridge_mqtt_lib:clientid_base([Name, "E"]), + %% : + Name = <<"abc012345678901234567890">>, + BaseId = emqx_bridge_mqtt_lib:clientid_base([Name]), ExpectedClientId = emqx_bridge_mqtt_lib:bytes23(BaseId, 1), test_egress_clientid(Name, ExpectedClientId). @@ -1049,7 +1049,8 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - emqx_utils_json:decode(Bridge) + emqx_utils_json:decode(Bridge), + #{expected_type => Type, expected_name => Name} ), emqx_bridge_resource:bridge_id(Type, Name). From 28de7c89c7c02bc6114fb62133e5ee11d2b8110d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 8 Jan 2024 17:24:55 -0300 Subject: [PATCH 42/62] feat: add `/sources*` HTTP APIs --- apps/emqx_bridge/src/emqx_bridge_resource.erl | 20 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 23 +- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 384 ++++++++++++++++-- .../src/proto/emqx_bridge_proto_v6.erl | 12 +- .../src/schema/emqx_bridge_v2_schema.erl | 239 ++++++++--- ...qx_bridge_v1_compatibility_layer_SUITE.erl | 2 +- .../test/emqx_bridge_v2_testlib.erl | 187 +++++---- .../emqx_bridge/test/emqx_bridge_v2_tests.erl | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 4 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 23 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 175 ++++++++ 11 files changed, 870 insertions(+), 201 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 0a870abb8..ec7a7431b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -23,6 +23,7 @@ bridge_to_resource_type/1, resource_id/1, resource_id/2, + resource_id/3, bridge_id/2, parse_bridge_id/1, parse_bridge_id/2, @@ -62,6 +63,9 @@ ?IS_BI_DIR_BRIDGE(TYPE) ). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + -if(?EMQX_RELEASE_EDITION == ee). bridge_to_resource_type(BridgeType) when is_binary(BridgeType) -> bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8)); @@ -85,11 +89,21 @@ bridge_impl_module(_BridgeType) -> undefined. -endif. resource_id(BridgeId) when is_binary(BridgeId) -> + resource_id_for_kind(?ROOT_KEY_ACTIONS, BridgeId). + +resource_id(BridgeType, BridgeName) -> + resource_id(?ROOT_KEY_ACTIONS, BridgeType, BridgeName). + +resource_id(ConfRootKey, BridgeType, BridgeName) -> + BridgeId = bridge_id(BridgeType, BridgeName), + resource_id_for_kind(ConfRootKey, BridgeId). + +resource_id_for_kind(ConfRootKey, BridgeId) when is_binary(BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, _Name] -> case emqx_bridge_v2:is_bridge_v2_type(Type) of true -> - emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(BridgeId); + emqx_bridge_v2:bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId); false -> <<"bridge:", BridgeId/binary>> end; @@ -97,10 +111,6 @@ resource_id(BridgeId) when is_binary(BridgeId) -> invalid_data(<<"should be of pattern {type}:{name}, but got ", BridgeId/binary>>) end. -resource_id(BridgeType, BridgeName) -> - BridgeId = bridge_id(BridgeType, BridgeName), - resource_id(BridgeId). - bridge_id(BridgeType, BridgeName) -> Name = bin(BridgeName), Type = bin(BridgeType), diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index fb8ae2e2e..67aeeca41 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -91,6 +91,7 @@ id/2, id/3, bridge_v1_is_valid/2, + bridge_v1_is_valid/3, extract_connector_id_from_bridge_v2_id/1 ]). @@ -128,6 +129,7 @@ %% Exception from the naming convention: bridge_v2_type_to_bridge_v1_type/2, bridge_v1_id_to_connector_resource_id/1, + bridge_v1_id_to_connector_resource_id/2, bridge_v1_enable_disable/3, bridge_v1_restart/2, bridge_v1_stop/2, @@ -567,7 +569,7 @@ connector_operation_helper(ConfRootKey, BridgeV2Type, Name, ConnectorOpFun, DoHe ConfRootKey, BridgeV2Type, Name, - lookup_conf(BridgeV2Type, Name), + lookup_conf(ConfRootKey, BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ). @@ -1191,8 +1193,11 @@ unpack_bridge_conf(Type, PackedConf, TopLevelConf) -> %% * The corresponding bridge v2 should exist %% * The connector for the bridge v2 should have exactly one channel bridge_v1_is_valid(BridgeV1Type, BridgeName) -> + bridge_v1_is_valid(?ROOT_KEY_ACTIONS, BridgeV1Type, BridgeName). + +bridge_v1_is_valid(ConfRootKey, BridgeV1Type, BridgeName) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), - case lookup_conf(BridgeV2Type, BridgeName) of + case lookup_conf(ConfRootKey, BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 true; @@ -1241,17 +1246,20 @@ bridge_v1_list_and_transform() -> bridge_v1_lookup_and_transform(ActionType, Name) -> case lookup_actions_or_sources(ActionType, Name) of - {ok, ConfRootName, + {ok, ConfRootKey, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), HasBridgeV1Equivalent = has_bridge_v1_equivalent(ActionType), - case HasBridgeV1Equivalent andalso ?MODULE:bridge_v1_is_valid(BridgeV1Type, Name) of + case + HasBridgeV1Equivalent andalso + ?MODULE:bridge_v1_is_valid(ConfRootKey, BridgeV1Type, Name) + of true -> ConnectorType = connector_type(ActionType), case emqx_connector:lookup(ConnectorType, ConnectorName) of {ok, Connector} -> bridge_v1_lookup_and_transform_helper( - ConfRootName, + ConfRootKey, BridgeV1Type, Name, ActionType, @@ -1718,11 +1726,14 @@ connector_has_channels(BridgeV2Type, ConnectorName) -> end. bridge_v1_id_to_connector_resource_id(BridgeId) -> + bridge_v1_id_to_connector_resource_id(?ROOT_KEY_ACTIONS, BridgeId). + +bridge_v1_id_to_connector_resource_id(ConfRootKey, BridgeId) -> case binary:split(BridgeId, <<":">>) of [Type, Name] -> BridgeV2Type = bin(bridge_v1_type_to_bridge_v2_type(Type)), ConnectorName = - case lookup_conf(BridgeV2Type, Name) of + case lookup_conf(ConfRootKey, BridgeV2Type, Name) of #{connector := Con} -> Con; {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 3f0d18fae..d4401cfd0 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -37,7 +37,7 @@ namespace/0 ]). -%% API callbacks +%% API callbacks : actions -export([ '/actions'/2, '/actions/:id'/2, @@ -49,6 +49,18 @@ '/actions_probe'/2, '/action_types'/2 ]). +%% API callbacks : sources +-export([ + '/sources'/2, + '/sources/:id'/2, + '/sources/:id/metrics'/2, + '/sources/:id/metrics/reset'/2, + '/sources/:id/enable/:enable'/2, + '/sources/:id/:operation'/2, + '/nodes/:node/sources/:id/:operation'/2, + '/sources_probe'/2, + '/source_types'/2 +]). %% BpAPI / RPC Targets -export([ @@ -81,13 +93,16 @@ end ). -namespace() -> "actions". +namespace() -> "actions_and_sources". api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ + %%============= + %% Actions + %%============= "/actions", "/actions/:id", "/actions/:id/enable/:enable", @@ -98,7 +113,21 @@ paths() -> "/actions/:id/metrics", "/actions/:id/metrics/reset", "/actions_probe", - "/action_types" + "/action_types", + %%============= + %% Sources + %%============= + "/sources", + "/sources/:id", + "/sources/:id/enable/:enable", + "/sources/:id/:operation", + "/nodes/:node/sources/:id/:operation", + %% %% Caveat: metrics paths must come *after* `/:operation', otherwise minirest will + %% %% try to match the latter first, trying to interpret `metrics' as an operation... + "/sources/:id/metrics", + "/sources/:id/metrics/reset", + "/sources_probe" + %% "/source_types" ]. error_schema(Code, Message) -> @@ -111,17 +140,28 @@ error_schema(Codes, Message, ExtraFields) when is_list(Message) -> error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) -> ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message). -get_response_body_schema() -> +actions_get_response_body_schema() -> emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:get_response(), - bridge_info_examples(get) + emqx_bridge_v2_schema:actions_get_response(), + bridge_info_examples(get, ?ROOT_KEY_ACTIONS) ). -bridge_info_examples(Method) -> - emqx_bridge_v2_schema:examples(Method). +sources_get_response_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_get_response(), + bridge_info_examples(get, ?ROOT_KEY_SOURCES) + ). -bridge_info_array_example(Method) -> - lists:map(fun(#{value := Config}) -> Config end, maps:values(bridge_info_examples(Method))). +bridge_info_examples(Method, ?ROOT_KEY_ACTIONS) -> + emqx_bridge_v2_schema:actions_examples(Method); +bridge_info_examples(Method, ?ROOT_KEY_SOURCES) -> + emqx_bridge_v2_schema:sources_examples(Method). + +bridge_info_array_example(Method, ConfRootKey) -> + lists:map( + fun(#{value := Config}) -> Config end, + maps:values(bridge_info_examples(Method, ConfRootKey)) + ). param_path_id() -> {id, @@ -195,6 +235,9 @@ param_path_enable() -> } )}. +%%================================================================================ +%% Actions +%%================================================================================ schema("/actions") -> #{ 'operationId' => '/actions', @@ -204,8 +247,8 @@ schema("/actions") -> description => ?DESC("desc_api1"), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - array(emqx_bridge_v2_schema:get_response()), - bridge_info_array_example(get) + array(emqx_bridge_v2_schema:actions_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_ACTIONS) ) } }, @@ -214,11 +257,11 @@ schema("/actions") -> summary => <<"Create bridge">>, description => ?DESC("desc_api2"), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ - 201 => get_response_body_schema(), + 201 => actions_get_response_body_schema(), 400 => error_schema('ALREADY_EXISTS', "Bridge already exists") } } @@ -232,7 +275,7 @@ schema("/actions/:id") -> description => ?DESC("desc_api3"), parameters => [param_path_id()], responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found") } }, @@ -242,11 +285,11 @@ schema("/actions/:id") -> description => ?DESC("desc_api4"), parameters => [param_path_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:put_request(), - bridge_info_examples(put) + emqx_bridge_v2_schema:actions_put_request(), + bridge_info_examples(put, ?ROOT_KEY_ACTIONS) ), responses => #{ - 200 => get_response_body_schema(), + 200 => actions_get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Bridge not found"), 400 => error_schema('BAD_REQUEST', "Update bridge failed") } @@ -371,8 +414,8 @@ schema("/actions_probe") -> desc => ?DESC("desc_api9"), summary => <<"Test creating bridge">>, 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_bridge_v2_schema:post_request(), - bridge_info_examples(post) + emqx_bridge_v2_schema:actions_post_request(), + bridge_info_examples(post, ?ROOT_KEY_ACTIONS) ), responses => #{ 204 => <<"Test bridge OK">>, @@ -389,12 +432,223 @@ schema("/action_types") -> summary => <<"List available action types">>, responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - array(emqx_bridge_v2_schema:types_sc()), + array(emqx_bridge_v2_schema:action_types_sc()), #{ <<"types">> => #{ summary => <<"Action types">>, - value => emqx_bridge_v2_schema:types() + value => emqx_bridge_v2_schema:action_types() + } + } + ) + } + } + }; +%%================================================================================ +%% Sources +%%================================================================================ +schema("/sources") -> + #{ + 'operationId' => '/sources', + get => #{ + tags => [<<"sources">>], + summary => <<"List sources">>, + description => ?DESC("desc_api1"), + responses => #{ + %% FIXME: examples + 200 => emqx_dashboard_swagger:schema_with_example( + array(emqx_bridge_v2_schema:sources_get_response()), + bridge_info_array_example(get, ?ROOT_KEY_SOURCES) + ) + } + }, + post => #{ + tags => [<<"sources">>], + summary => <<"Create source">>, + description => ?DESC("desc_api2"), + %% FIXME: examples + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 201 => sources_get_response_body_schema(), + 400 => error_schema('ALREADY_EXISTS', "Source already exists") + } + } + }; +schema("/sources/:id") -> + #{ + 'operationId' => '/sources/:id', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source">>, + description => ?DESC("desc_api3"), + parameters => [param_path_id()], + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + }, + put => #{ + tags => [<<"sources">>], + summary => <<"Update source">>, + description => ?DESC("desc_api4"), + parameters => [param_path_id()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_put_request(), + bridge_info_examples(put, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 200 => sources_get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Source not found"), + 400 => error_schema('BAD_REQUEST', "Update source failed") + } + }, + delete => #{ + tags => [<<"sources">>], + summary => <<"Delete source">>, + description => ?DESC("desc_api5"), + parameters => [param_path_id(), param_qs_delete_cascade()], + responses => #{ + 204 => <<"Source deleted">>, + 400 => error_schema( + 'BAD_REQUEST', + "Cannot delete bridge while active rules are defined for this source", + [{rules, mk(array(string()), #{desc => "Dependent Rule IDs"})}] + ), + 404 => error_schema('NOT_FOUND', "Source not found"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/metrics") -> + #{ + 'operationId' => '/sources/:id/metrics', + get => #{ + tags => [<<"sources">>], + summary => <<"Get source metrics">>, + description => ?DESC("desc_bridge_metrics"), + parameters => [param_path_id()], + responses => #{ + 200 => emqx_bridge_schema:metrics_fields(), + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/metrics/reset") -> + #{ + 'operationId' => '/sources/:id/metrics/reset', + put => #{ + tags => [<<"sources">>], + summary => <<"Reset source metrics">>, + description => ?DESC("desc_api6"), + parameters => [param_path_id()], + responses => #{ + 204 => <<"Reset success">>, + 404 => error_schema('NOT_FOUND', "Source not found") + } + } + }; +schema("/sources/:id/enable/:enable") -> + #{ + 'operationId' => '/sources/:id/enable/:enable', + put => + #{ + tags => [<<"sources">>], + summary => <<"Enable or disable bridge">>, + desc => ?DESC("desc_enable_bridge"), + parameters => [param_path_id(), param_path_enable()], + responses => + #{ + 204 => <<"Success">>, + 404 => error_schema( + 'NOT_FOUND', "Bridge not found or invalid operation" + ), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources/:id/:operation") -> + #{ + 'operationId' => '/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge">>, + description => ?DESC("desc_api7"), + parameters => [ + param_path_id(), + param_path_operation_cluster() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', "Problem with configuration of external service" + ), + 404 => error_schema('NOT_FOUND', "Bridge not found or invalid operation"), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/nodes/:node/sources/:id/:operation") -> + #{ + 'operationId' => '/nodes/:node/sources/:id/:operation', + post => #{ + tags => [<<"sources">>], + summary => <<"Manually start a bridge on a given node">>, + description => ?DESC("desc_api8"), + parameters => [ + param_path_node(), + param_path_id(), + param_path_operation_on_node() + ], + responses => #{ + 204 => <<"Operation success">>, + 400 => error_schema( + 'BAD_REQUEST', + "Problem with configuration of external service or bridge not enabled" + ), + 404 => error_schema( + 'NOT_FOUND', "Bridge or node not found or invalid operation" + ), + 501 => error_schema('NOT_IMPLEMENTED', "Not Implemented"), + 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") + } + } + }; +schema("/sources_probe") -> + #{ + 'operationId' => '/sources_probe', + post => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api9"), + summary => <<"Test creating bridge">>, + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_v2_schema:sources_post_request(), + bridge_info_examples(post, ?ROOT_KEY_SOURCES) + ), + responses => #{ + 204 => <<"Test bridge OK">>, + 400 => error_schema(['TEST_FAILED'], "bridge test failed") + } + } + }; +schema("/source_types") -> + #{ + 'operationId' => '/source_types', + get => #{ + tags => [<<"sources">>], + desc => ?DESC("desc_api10"), + summary => <<"List available source types">>, + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_examples( + array(emqx_bridge_v2_schema:action_types_sc()), + #{ + <<"types">> => + #{ + summary => <<"Source types">>, + value => emqx_bridge_v2_schema:action_types() } } ) @@ -402,6 +656,12 @@ schema("/action_types") -> } }. +%%------------------------------------------------------------------------------ +%% Thin Handlers +%%------------------------------------------------------------------------------ +%%================================================================================ +%% Actions +%%================================================================================ '/actions'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> handle_create(?ROOT_KEY_ACTIONS, BridgeType, BridgeName, Conf0); '/actions'(get, _Params) -> @@ -439,7 +699,48 @@ schema("/action_types") -> handle_probe(?ROOT_KEY_ACTIONS, Request). '/action_types'(get, _Request) -> - ?OK(emqx_bridge_v2_schema:types()). + ?OK(emqx_bridge_v2_schema:action_types()). +%%================================================================================ +%% Sources +%%================================================================================ +'/sources'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) -> + handle_create(?ROOT_KEY_SOURCES, BridgeType, BridgeName, Conf0); +'/sources'(get, _Params) -> + handle_list(?ROOT_KEY_SOURCES). + +'/sources/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName, 200)); +'/sources/:id'(put, #{bindings := #{id := Id}, body := Conf0}) -> + handle_update(?ROOT_KEY_SOURCES, Id, Conf0); +'/sources/:id'(delete, #{bindings := #{id := Id}, query_string := Qs}) -> + handle_delete(?ROOT_KEY_SOURCES, Id, Qs). + +'/sources/:id/metrics'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(?ROOT_KEY_SOURCES, BridgeType, BridgeName)). + +'/sources/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> + handle_reset_metrics(?ROOT_KEY_SOURCES, Id). + +'/sources/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> + handle_disable_enable(?ROOT_KEY_SOURCES, Id, Enable). + +'/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op} +}) -> + handle_operation(?ROOT_KEY_SOURCES, Id, Op). + +'/nodes/:node/sources/:id/:operation'(post, #{ + bindings := + #{id := Id, operation := Op, node := Node} +}) -> + handle_node_operation(?ROOT_KEY_SOURCES, Node, Id, Op). + +'/sources_probe'(post, Request) -> + handle_probe(?ROOT_KEY_SOURCES, Request). + +'/source_types'(get, _Request) -> + ?OK(emqx_bridge_v2_schema:source_types()). %%------------------------------------------------------------------------------ %% Handlers @@ -451,7 +752,7 @@ handle_list(ConfRootKey) -> case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ - [format_resource(Data, Node) || Data <- Bridges] + [format_resource(ConfRootKey, Data, Node) || Data <- Bridges] || {Node, Bridges} <- lists:zip(Nodes, NodeBridges) ], ?OK(zip_bridges(AllBridges)); @@ -574,7 +875,12 @@ handle_node_operation(ConfRootKey, Node, Id, Op) -> ). handle_probe(ConfRootKey, Request) -> - RequestMeta = #{module => ?MODULE, method => post, path => "/actions_probe"}, + Path = + case ConfRootKey of + ?ROOT_KEY_ACTIONS -> "/actions_probe"; + ?ROOT_KEY_SOURCES -> "/sources_probe" + end, + RequestMeta = #{module => ?MODULE, method => post, path => Path}, case emqx_dashboard_swagger:filter_check_request_and_translate_body(Request, RequestMeta) of {ok, #{body := #{<<"type">> := Type} = Params}} -> Params1 = maybe_deobfuscate_bridge_probe(Params), @@ -664,8 +970,8 @@ get_metrics_from_all_nodes(ConfRootKey, Type, Name) -> ?INTERNAL_ERROR(Reason) end. -operation_func(all, start) -> v2_start_bridge_to_all_nodes_v6; -operation_func(_Node, start) -> v2_start_bridge_to_node_v6; +operation_func(all, start) -> v2_start_bridge_on_all_nodes_v6; +operation_func(_Node, start) -> v2_start_bridge_on_node_v6; operation_func(all, lookup) -> v2_lookup_from_all_nodes_v6; operation_func(all, list) -> v2_list_bridges_on_nodes_v6; operation_func(all, get_metrics) -> v2_get_metrics_from_all_nodes_v6. @@ -825,7 +1131,7 @@ aggregate_status(AllStatus) -> %% RPC Target lookup_from_local_node(BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resource(Res, node())}; + {ok, Res} -> {ok, format_resource(?ROOT_KEY_ACTIONS, Res, node())}; Error -> Error end. @@ -833,7 +1139,7 @@ lookup_from_local_node(BridgeType, BridgeName) -> -spec lookup_from_local_node_v6(emqx_bridge_v2:root_cfg_key(), _, _) -> _. lookup_from_local_node_v6(ConfRootKey, BridgeType, BridgeName) -> case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of - {ok, Res} -> {ok, format_resource(Res, node())}; + {ok, Res} -> {ok, format_resource(ConfRootKey, Res, node())}; Error -> Error end. @@ -847,6 +1153,7 @@ get_metrics_from_local_node_v6(ConfRootKey, Type, Name) -> %% resource format_resource( + ConfRootKey, #{ type := Type, name := Name, @@ -857,7 +1164,7 @@ format_resource( }, Node ) -> - RawConf = fill_defaults(Type, RawConf0), + RawConf = fill_defaults(ConfRootKey, Type, RawConf0), redact( maps:merge( RawConf#{ @@ -988,17 +1295,18 @@ aggregate_metrics( M17 + N17 ). -fill_defaults(Type, RawConf) -> - PackedConf = pack_bridge_conf(Type, RawConf), +fill_defaults(ConfRootKey, Type, RawConf) -> + PackedConf = pack_bridge_conf(ConfRootKey, Type, RawConf), FullConf = emqx_config:fill_defaults(emqx_bridge_v2_schema, PackedConf, #{}), - unpack_bridge_conf(Type, FullConf). + unpack_bridge_conf(ConfRootKey, Type, FullConf). -pack_bridge_conf(Type, RawConf) -> - #{<<"actions">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. +pack_bridge_conf(ConfRootKey, Type, RawConf) -> + #{bin(ConfRootKey) => #{bin(Type) => #{<<"foo">> => RawConf}}}. -unpack_bridge_conf(Type, PackedConf) -> +unpack_bridge_conf(ConfRootKey, Type, PackedConf) -> + ConfRootKeyBin = bin(ConfRootKey), TypeBin = bin(Type), - #{<<"actions">> := Bridges} = PackedConf, + #{ConfRootKeyBin := Bridges} = PackedConf, #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), RawConf. diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl index d6fe68466..fbcef8b5c 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v6.erl @@ -35,8 +35,8 @@ v2_lookup_from_all_nodes_v6/4, v2_list_bridges_on_nodes_v6/2, v2_get_metrics_from_all_nodes_v6/4, - v2_start_bridge_to_node_v6/4, - v2_start_bridge_to_all_nodes_v6/4 + v2_start_bridge_on_node_v6/4, + v2_start_bridge_on_all_nodes_v6/4 ]). -include_lib("emqx/include/bpapi.hrl"). @@ -173,9 +173,9 @@ v2_get_metrics_from_all_nodes_v6(Nodes, ConfRootKey, ActionType, ActionName) -> ?TIMEOUT ). --spec v2_start_bridge_to_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> +-spec v2_start_bridge_on_all_nodes_v6([node()], emqx_bridge_v2:root_cfg_key(), key(), key()) -> emqx_rpc:erpc_multicall(ok). -v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> +v2_start_bridge_on_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> erpc:multicall( Nodes, emqx_bridge_v2, @@ -184,9 +184,9 @@ v2_start_bridge_to_all_nodes_v6(Nodes, ConfRootKey, BridgeType, BridgeName) -> ?TIMEOUT ). --spec v2_start_bridge_to_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> +-spec v2_start_bridge_on_node_v6(node(), emqx_bridge_v2:root_cfg_key(), key(), key()) -> term(). -v2_start_bridge_to_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> +v2_start_bridge_on_node_v6(Node, ConfRootKey, BridgeType, BridgeName) -> rpc:call( Node, emqx_bridge_v2, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 28017f814..ec9314fd2 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -28,21 +28,31 @@ -export([roots/0, fields/1, desc/1, namespace/0, tags/0]). -export([ - get_response/0, - put_request/0, - post_request/0, - examples/1, + actions_get_response/0, + actions_put_request/0, + actions_post_request/0, + actions_examples/1, action_values/4 ]). +-export([ + sources_get_response/0, + sources_put_request/0, + sources_post_request/0, + sources_examples/1, + source_values/4 +]). + %% Exported for mocking %% TODO: refactor emqx_bridge_v1_compatibility_layer_SUITE so we don't need to %% export this -export([ - registered_api_schemas/1 + registered_actions_api_schemas/1, + registered_sources_api_schemas/1 ]). --export([types/0, types_sc/0]). +-export([action_types/0, action_types_sc/0]). +-export([source_types/0, source_types_sc/0]). -export([resource_opts_fields/0, resource_opts_fields/1]). -export([ @@ -58,33 +68,140 @@ -export([actions_convert_from_connectors/1]). --export_type([action_type/0]). +-export_type([action_type/0, source_type/0]). %% Should we explicitly list them here so dialyzer may be more helpful? -type action_type() :: atom(). +-type source_type() :: atom(). +-type http_method() :: get | post | put. +-type schema_example_map() :: #{atom() => term()}. %%====================================================================================== %% For HTTP APIs -get_response() -> - api_schema("get"). +%%====================================================================================== -put_request() -> - api_schema("put"). +%%--------------------------------------------- +%% Actions +%%--------------------------------------------- -post_request() -> - api_schema("post"). +actions_get_response() -> + actions_api_schema("get"). -api_schema(Method) -> - APISchemas = ?MODULE:registered_api_schemas(Method), +actions_put_request() -> + actions_api_schema("put"). + +actions_post_request() -> + actions_api_schema("post"). + +actions_api_schema(Method) -> + APISchemas = ?MODULE:registered_actions_api_schemas(Method), hoconsc:union(bridge_api_union(APISchemas)). -registered_api_schemas(Method) -> +registered_actions_api_schemas(Method) -> RegisteredSchemas = emqx_action_info:registered_schema_modules_actions(), [ api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_bridge_v2") || {BridgeV2Type, SchemaModule} <- RegisteredSchemas ]. +-spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +action_values(Method, ActionType, ConnectorType, ActionValues) -> + ActionTypeBin = atom_to_binary(ActionType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", ActionTypeBin/binary, " action">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + ActionValues, + method_values(action, Method, ActionType) + ] + ). + +actions_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Sources +%%--------------------------------------------- + +sources_get_response() -> + sources_api_schema("get"). + +sources_put_request() -> + sources_api_schema("put"). + +sources_post_request() -> + sources_api_schema("post"). + +sources_api_schema(Method) -> + APISchemas = ?MODULE:registered_sources_api_schemas(Method), + hoconsc:union(bridge_api_union(APISchemas)). + +registered_sources_api_schemas(Method) -> + RegisteredSchemas = emqx_action_info:registered_schema_modules_sources(), + [ + api_ref(SchemaModule, atom_to_binary(BridgeV2Type), Method ++ "_source") + || {BridgeV2Type, SchemaModule} <- RegisteredSchemas + ]. + +-spec source_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). +source_values(Method, SourceType, ConnectorType, SourceValues) -> + SourceTypeBin = atom_to_binary(SourceType), + ConnectorTypeBin = atom_to_binary(ConnectorType), + lists:foldl( + fun(M1, M2) -> + maps:merge(M1, M2) + end, + #{ + enable => true, + description => <<"My example ", SourceTypeBin/binary, " source">>, + connector => <>, + resource_opts => #{ + health_check_interval => "30s" + } + }, + [ + SourceValues, + method_values(source, Method, SourceType) + ] + ). + +sources_examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_sources()], + lists:foldl(Fun, #{}, SchemaModules). + +%%--------------------------------------------- +%% Common helpers +%%--------------------------------------------- + api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. @@ -111,41 +228,17 @@ bridge_api_union(Refs) -> end end. --type http_method() :: get | post | put. --type schema_example_map() :: #{atom() => term()}. - --spec action_values(http_method(), atom(), atom(), schema_example_map()) -> schema_example_map(). -action_values(Method, ActionType, ConnectorType, ActionValues) -> - ActionTypeBin = atom_to_binary(ActionType), - ConnectorTypeBin = atom_to_binary(ConnectorType), - lists:foldl( - fun(M1, M2) -> - maps:merge(M1, M2) - end, - #{ - enable => true, - description => <<"My example ", ActionTypeBin/binary, " action">>, - connector => <>, - resource_opts => #{ - health_check_interval => "30s" - } - }, - [ - ActionValues, - method_values(Method, ActionType) - ] - ). - --spec method_values(http_method(), atom()) -> schema_example_map(). -method_values(post, Type) -> +-spec method_values(action | source, http_method(), atom()) -> schema_example_map(). +method_values(Kind, post, Type) -> + KindBin = atom_to_binary(Kind), TypeBin = atom_to_binary(Type), #{ - name => <>, + name => <>, type => TypeBin }; -method_values(get, Type) -> +method_values(Kind, get, Type) -> maps:merge( - method_values(post, Type), + method_values(Kind, post, Type), #{ status => <<"connected">>, node_status => [ @@ -156,7 +249,7 @@ method_values(get, Type) -> ] } ); -method_values(put, _Type) -> +method_values(_Kind, put, _Type) -> #{}. api_fields("get_bridge_v2", Type, Fields) -> @@ -175,16 +268,33 @@ api_fields("post_bridge_v2", Type, Fields) -> ] ); api_fields("put_bridge_v2", _Type, Fields) -> + Fields; +api_fields("get_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + emqx_bridge_schema:status_fields(), + Fields + ] + ); +api_fields("post_source", Type, Fields) -> + lists:append( + [ + emqx_bridge_schema:type_and_name_fields(Type), + Fields + ] + ); +api_fields("put_source", _Type, Fields) -> Fields. %%====================================================================================== %% HOCON Schema Callbacks %%====================================================================================== -namespace() -> "actions". +namespace() -> "actions_and_sources". tags() -> - [<<"Actions">>]. + [<<"Actions">>, <<"Sources">>]. -dialyzer({nowarn_function, roots/0}). @@ -231,13 +341,21 @@ desc(resource_opts) -> desc(_) -> undefined. --spec types() -> [action_type()]. -types() -> +-spec action_types() -> [action_type()]. +action_types() -> proplists:get_keys(?MODULE:fields(actions)). --spec types_sc() -> ?ENUM([action_type()]). -types_sc() -> - hoconsc:enum(types()). +-spec action_types_sc() -> ?ENUM([action_type()]). +action_types_sc() -> + hoconsc:enum(action_types()). + +-spec source_types() -> [source_type()]. +source_types() -> + proplists:get_keys(?MODULE:fields(sources)). + +-spec source_types_sc() -> ?ENUM([source_type()]). +source_types_sc() -> + hoconsc:enum(source_types()). resource_opts_fields() -> resource_opts_fields(_Overrides = []). @@ -268,19 +386,6 @@ resource_opts_fields(Overrides) -> emqx_resource_schema:create_opts(Overrides) ). -examples(Method) -> - MergeFun = - fun(Example, Examples) -> - maps:merge(Examples, Example) - end, - Fun = - fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), - lists:foldl(MergeFun, Examples, ConnectorExamples) - end, - SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_actions()], - lists:foldl(Fun, #{}, SchemaModules). - top_level_common_action_keys() -> [ <<"connector">>, diff --git a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl index dadc0a09c..b67791cb3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl @@ -104,7 +104,7 @@ setup_mocks() -> catch meck:new(emqx_bridge_v2_schema, MeckOpts), meck:expect( emqx_bridge_v2_schema, - registered_api_schemas, + registered_actions_api_schemas, 1, fun(Method) -> [{bridge_type_bin(), hoconsc:ref(?MODULE, "api_v2_" ++ Method)}] diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index f7dd74161..88788d6e2 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -12,6 +12,9 @@ -import(emqx_common_test_helpers, [on_exit/1]). +-define(ROOT_KEY_ACTIONS, actions). +-define(ROOT_KEY_SOURCES, sources). + %% ct setup helpers init_per_suite(Config, Apps) -> @@ -152,6 +155,49 @@ create_bridge(Config, Overrides) -> ct:pal("creating bridge with config: ~p", [BridgeConfig]), emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig). +get_ct_config_with_fallback(Config, [Key]) -> + ?config(Key, Config); +get_ct_config_with_fallback(Config, [Key | Rest]) -> + case ?config(Key, Config) of + undefined -> + get_ct_config_with_fallback(Config, Rest); + X -> + X + end. + +get_config_by_kind(Config, Overrides) -> + Kind = ?config(bridge_kind, Config), + get_config_by_kind(Kind, Config, Overrides). + +get_config_by_kind(Kind, Config, Overrides) -> + case Kind of + action -> + %% TODO: refactor tests to use action_type... + ActionType = get_ct_config_with_fallback(Config, [action_type, bridge_type]), + ActionName = get_ct_config_with_fallback(Config, [action_name, bridge_name]), + ActionConfig0 = get_ct_config_with_fallback(Config, [action_config, bridge_config]), + ActionConfig = emqx_utils_maps:deep_merge(ActionConfig0, Overrides), + #{type => ActionType, name => ActionName, config => ActionConfig}; + source -> + SourceType = ?config(source_type, Config), + SourceName = ?config(source_name, Config), + SourceConfig0 = ?config(source_config, Config), + SourceConfig = emqx_utils_maps:deep_merge(SourceConfig0, Overrides), + #{type => SourceType, name => SourceName, config => SourceConfig} + end. + +api_path_root(Kind) -> + case Kind of + action -> "actions"; + source -> "sources" + end. + +conf_root_key(Kind) -> + case Kind of + action -> ?ROOT_KEY_ACTIONS; + source -> ?ROOT_KEY_SOURCES + end. + maybe_json_decode(X) -> case emqx_utils_json:safe_decode(X, [return_maps]) of {ok, Decoded} -> Decoded; @@ -218,26 +264,26 @@ create_bridge_api(Config) -> create_bridge_api(Config, _Overrides = #{}). create_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - {ok, {{_, 201, _}, _, _}} = create_connector_api(Config), + create_kind_api(Config, Overrides). - Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("creating bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; - Error -> - Error - end, - ct:pal("bridge create result: ~p", [Res]), +create_kind_api(Config) -> + create_kind_api(Config, _Overrides = #{}). + +create_kind_api(Config, Overrides) -> + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, Overrides), + Params = BridgeConfig#{<<"type">> => Type, <<"name">> => Name}, + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot]), + ct:pal("creating bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge create (~s, http) result:\n ~p", [Kind, Res]), Res. create_connector_api(Config) -> @@ -288,27 +334,29 @@ update_bridge_api(Config) -> update_bridge_api(Config, _Overrides = #{}). update_bridge_api(Config, Overrides) -> - BridgeType = ?config(bridge_type, Config), - Name = ?config(bridge_name, Config), - BridgeConfig0 = ?config(bridge_config, Config), - BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), - BridgeId = emqx_bridge_resource:bridge_id(BridgeType, Name), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("updating bridge (via http): ~p", [BridgeConfig]), - Res = - case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, BridgeConfig, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; - Error -> Error - end, - ct:pal("bridge update result: ~p", [Res]), + Kind = proplists:get_value(bridge_kind, Config, action), + #{ + type := Type, + name := Name, + config := Params + } = get_config_by_kind(Kind, Config, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId]), + ct:pal("updating bridge (~s, http):\n ~p", [Kind, Params]), + Method = put, + Res = request(Method, Path, Params), + ct:pal("update bridge (~s, http) result:\n ~p", [Kind, Res]), Res. op_bridge_api(Op, BridgeType, BridgeName) -> + op_bridge_api(_Kind = action, Op, BridgeType, BridgeName). + +op_bridge_api(Kind, Op, BridgeType, BridgeName) -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), - Path = emqx_mgmt_api_test_util:api_path(["actions", BridgeId, Op]), - ct:pal("calling bridge ~p (via http): ~p", [BridgeId, Op]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot, BridgeId, Op]), + ct:pal("calling bridge ~p (~s, http):\n ~p", [BridgeId, Kind, Op]), Method = post, Params = [], Res = request(Method, Path, Params), @@ -326,17 +374,16 @@ probe_bridge_api(Config, Overrides) -> probe_bridge_api(BridgeType, BridgeName, BridgeConfig). probe_bridge_api(BridgeType, BridgeName, BridgeConfig) -> + probe_bridge_api(action, BridgeType, BridgeName, BridgeConfig). + +probe_bridge_api(Kind, BridgeType, BridgeName, BridgeConfig) -> Params = BridgeConfig#{<<"type">> => BridgeType, <<"name">> => BridgeName}, - Path = emqx_mgmt_api_test_util:api_path(["actions_probe"]), - AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - Opts = #{return_all => true}, - ct:pal("probing bridge (via http): ~p", [Params]), - Res = - case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of - {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; - Error -> Error - end, - ct:pal("bridge probe result: ~p", [Res]), + PathRoot = api_path_root(Kind), + Path = emqx_mgmt_api_test_util:api_path([PathRoot ++ "_probe"]), + ct:pal("probing bridge (~s, http):\n ~p", [Kind, Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("bridge probe (~s, http) result:\n ~p", [Kind, Res]), Res. list_bridges_http_api_v1() -> @@ -353,6 +400,13 @@ list_actions_http_api() -> ct:pal("list actions (http v2) result:\n ~p", [Res]), Res. +list_sources_http_api() -> + Path = emqx_mgmt_api_test_util:api_path(["sources"]), + ct:pal("list sources (http v2)"), + Res = request(get, Path, _Params = []), + ct:pal("list sources (http v2) result:\n ~p", [Res]), + Res. + list_connectors_http_api() -> Path = emqx_mgmt_api_test_util:api_path(["connectors"]), ct:pal("list connectors"), @@ -506,13 +560,6 @@ t_create_via_http(Config) -> begin ?assertMatch({ok, _}, create_bridge_api(Config)), - %% lightweight matrix testing some configs - ?assertMatch( - {ok, _}, - update_bridge_api( - Config - ) - ), ?assertMatch( {ok, _}, update_bridge_api( @@ -526,23 +573,26 @@ t_create_via_http(Config) -> ok. t_start_stop(Config, StopTracePoint) -> - BridgeType = ?config(bridge_type, Config), - BridgeName = ?config(bridge_name, Config), - BridgeConfig = ?config(bridge_config, Config), + Kind = proplists:get_value(bridge_kind, Config, action), ConnectorName = ?config(connector_name, Config), ConnectorType = ?config(connector_type, Config), - ConnectorConfig = ?config(connector_config, Config), + #{ + type := Type, + name := Name, + config := BridgeConfig + } = get_config_by_kind(Kind, Config, _Overrides = #{}), ?assertMatch( - {ok, _}, - emqx_connector:create(ConnectorType, ConnectorName, ConnectorConfig) + {ok, {{_, 201, _}, _, _}}, + create_connector_api(Config) ), ?check_trace( begin ProbeRes0 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), @@ -550,8 +600,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsBefore = erlang:system_info(atom_count), %% Probe again; shouldn't have created more atoms. ProbeRes1 = probe_bridge_api( - BridgeType, - BridgeName, + Kind, + Type, + Name, BridgeConfig ), @@ -559,9 +610,9 @@ t_start_stop(Config, StopTracePoint) -> AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), - ?assertMatch({ok, _}, emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig)), + ?assertMatch({ok, _}, create_kind_api(Config)), - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + ResourceId = emqx_bridge_resource:resource_id(conf_root_key(Kind), Type, Name), %% Since the connection process is async, we give it some time to %% stabilize and avoid flakiness. @@ -574,7 +625,7 @@ t_start_stop(Config, StopTracePoint) -> %% `start` bridge to trigger `already_started` ?assertMatch( {ok, {{_, 204, _}, _Headers, []}}, - emqx_bridge_v2_testlib:op_bridge_api("start", BridgeType, BridgeName) + op_bridge_api(Kind, "start", Type, Name) ), ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), @@ -624,10 +675,10 @@ t_start_stop(Config, StopTracePoint) -> ) ), - ok + #{resource_id => ResourceId} end, - fun(Trace) -> - ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), + fun(Res, Trace) -> + #{resource_id := ResourceId} = Res, %% one for each probe, one for real ?assertMatch( [_, _, #{instance_id := ResourceId}], diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index e90100995..c64b1f2cb 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -108,7 +108,7 @@ connector_resource_opts_test() -> ok. actions_api_spec_post_fields_test() -> - ?UNION(Union) = emqx_bridge_v2_schema:post_request(), + ?UNION(Union) = emqx_bridge_v2_schema:actions_post_request(), Schemas = lists:map( fun(?R_REF(SchemaMod, StructName)) -> diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index dbdf68ef1..18af6ee11 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -237,7 +237,9 @@ on_stop(ResourceId, State) -> ets:delete(TopicToHandlerIndex) end, Allocated = emqx_resource:get_allocated_resources(ResourceId), - ok = stop_helper(Allocated). + ok = stop_helper(Allocated), + ?tp(mqtt_connector_stopped, #{instance_id => ResourceId}), + ok. stop_helper(#{pool_name := PoolName}) -> emqx_resource_pool:stop(PoolName). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 6d075334a..f765581f9 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -27,6 +27,9 @@ conn_bridge_examples/1 ]). +-define(ACTION_TYPE, mqtt). +-define(SOURCE_TYPE, mqtt). + %%====================================================================================== %% Hocon Schema Definitions namespace() -> "bridge_mqtt_publisher". @@ -86,14 +89,18 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, emqx_bridge_v2_schema:resource_opts_fields() ); -fields("get_connector") -> - emqx_bridge_mqtt_connector_schema:fields("config_connector"); -fields("get_bridge_v2") -> - fields("mqtt_publisher_action"); -fields("post_bridge_v2") -> - fields("mqtt_publisher_action") ++ emqx_bridge_schema:type_and_name_fields(mqtt); -fields("put_bridge_v2") -> - fields("mqtt_publisher_action"); +fields(Field) when + Field == "get_bridge_v2"; + Field == "post_bridge_v2"; + Field == "put_bridge_v2" +-> + emqx_bridge_v2_schema:api_fields(Field, ?ACTION_TYPE, fields("mqtt_publisher_action")); +fields(Field) when + Field == "get_source"; + Field == "post_source"; + Field == "put_source" +-> + emqx_bridge_v2_schema:api_fields(Field, ?SOURCE_TYPE, fields("mqtt_subscriber_source")); fields(What) -> error({emqx_bridge_mqtt_pubsub_schema, missing_field_handler, What}). %% v2: api schema diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl new file mode 100644 index 000000000..fde15a1b6 --- /dev/null +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -0,0 +1,175 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_bridge_mqtt_v2_subscriber_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_mqtt, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, Api} = emqx_common_test_http:create_default_app(), + [ + {apps, Apps}, + {api, Api} + | Config + ]. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), + ok. + +init_per_testcase(TestCase, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + Name = iolist_to_binary([atom_to_binary(TestCase), UniqueNum]), + ConnectorConfig = connector_config(), + SourceConfig = source_config(#{connector => Name}), + [ + {bridge_kind, source}, + {source_type, mqtt}, + {source_name, Name}, + {source_config, SourceConfig}, + {connector_type, mqtt}, + {connector_name, Name}, + {connector_config, ConnectorConfig} + | Config + ]. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +connector_config() -> + %% !!!!!!!!!!!! FIXME!!!!!! add more fields ("server_configs") + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + CommonConfig = + #{ + <<"enable">> => true, + <<"connector">> => <<"please override">>, + <<"parameters">> => + #{ + <<"remote">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + }, + <<"local">> => + #{ + <<"topic">> => <<"local/topic">>, + <<"qos">> => 2, + <<"retain">> => false, + <<"payload">> => <<"${payload}">> + } + }, + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"batch_time">> => <<"0ms">>, + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"15s">>, + <<"inflight_window">> => 100, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"45s">>, + <<"resume_interval">> => <<"15s">>, + <<"worker_pool_size">> => <<"1">> + } + }, + maps:merge(CommonConfig, Overrides). + +replace(Key, Value, Proplist) -> + lists:keyreplace(Key, 1, Proplist, {Key, Value}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_create_via_http(Config) -> + ConnectorName = ?config(connector_name, Config), + ok = emqx_bridge_v2_testlib:t_create_via_http(Config), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"enable">> := true, + <<"status">> := <<"connected">> + } + ]}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + NewSourceName = <<"my_other_source">>, + {ok, {{_, 201, _}, _, _}} = + emqx_bridge_v2_testlib:create_kind_api( + replace(source_name, NewSourceName, Config) + ), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{<<"connector">> := ConnectorName}, + #{<<"connector">> := ConnectorName} + ]}}, + emqx_bridge_v2_testlib:list_sources_http_api() + ), + ?assertMatch( + {ok, {{_, 200, _}, _, []}}, + emqx_bridge_v2_testlib:list_bridges_http_api_v1() + ), + ok. + +t_start_stop(Config) -> + ok = emqx_bridge_v2_testlib:t_start_stop(Config, mqtt_connector_stopped), + ok. From cc24fe6e933dbd563a3ddd7273cb09dda32396f8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Jan 2024 15:12:10 -0300 Subject: [PATCH 43/62] feat(mqtt_consumer): add support for rule engine `FROM` --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 13 ++- .../test/emqx_bridge_v2_testlib.erl | 17 ++++ .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 3 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 96 +++++++++++++++++-- .../emqx_connector/src/emqx_connector_api.erl | 7 +- .../src/emqx_rule_actions.erl | 6 ++ .../emqx_rule_engine/src/emqx_rule_engine.erl | 9 +- 8 files changed, 135 insertions(+), 18 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 67aeeca41..32034f774 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -54,6 +54,7 @@ check_deps_and_remove/3, check_deps_and_remove/4 ]). +-export([lookup_action/2, lookup_source/2]). %% Operations @@ -222,6 +223,12 @@ unload_bridges(ConfRooKey) -> lookup(Type, Name) -> lookup(?ROOT_KEY_ACTIONS, Type, Name). +lookup_action(Type, Name) -> + lookup(?ROOT_KEY_ACTIONS, Type, Name). + +lookup_source(Type, Name) -> + lookup(?ROOT_KEY_SOURCES, Type, Name). + -spec lookup(root_cfg_key(), bridge_v2_type(), bridge_v2_name()) -> {ok, bridge_v2_info()} | {error, not_found}. lookup(ConfRootName, Type, Name) -> @@ -900,9 +907,11 @@ do_get_matched_bridge_id(Topic, Filter, BType, BName, Acc) -> parse_id(Id) -> case binary:split(Id, <<":">>, [global]) of [Type, Name] -> - {Type, Name}; + #{kind => undefined, type => Type, name => Name}; [<<"action">>, Type, Name | _] -> - {Type, Name}; + #{kind => action, type => Type, name => Name}; + [<<"source">>, Type, Name | _] -> + #{kind => source, type => Type, name => Name}; _X -> error({error, iolist_to_binary(io_lib:format("Invalid id: ~p", [Id]))}) end. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 88788d6e2..7fef33115 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -442,6 +442,23 @@ try_decode_error(Body0) -> Body0 end. +create_rule_api(Opts) -> + #{ + sql := SQL, + actions := RuleActions + } = Opts, + Params = #{ + enable => true, + sql => SQL, + actions => RuleActions + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + ct:pal("create rule:\n ~p", [Params]), + Method = post, + Res = request(Method, Path, Params), + ct:pal("create rule results:\n ~p", [Res]), + Res. + create_rule_and_action_http(BridgeType, RuleTopic, Config) -> create_rule_and_action_http(BridgeType, RuleTopic, Config, _Opts = #{}). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 7caab1d87..459e259d2 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -135,7 +135,7 @@ create_producers_for_bridge_v2( KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), - {_BridgeType, BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), + #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), IsDryRun = case TestIdStart of diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index f765581f9..b4c2b63ba 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -82,6 +82,7 @@ fields("mqtt_subscriber_source") -> fields(ingress_parameters) -> Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), Fields1 = proplists:delete(pool_size, Fields0), + %% FIXME: should we make `local` hidden? Fields1; fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], @@ -120,8 +121,6 @@ desc(ingress_parameters) -> ?DESC(ingress_parameters); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; -desc("config_connector") -> - ?DESC("desc_config"); desc("http_action") -> ?DESC("desc_config"); desc("parameters_opts") -> diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index fde15a1b6..5569a826b 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -18,6 +18,7 @@ -compile(export_all). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("stdlib/include/assert.hrl"). -include_lib("emqx/include/asserts.hrl"). @@ -75,6 +76,11 @@ init_per_testcase(TestCase, Config) -> | Config ]. +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), + ok. + %%------------------------------------------------------------------------------ %% Helper fns %%------------------------------------------------------------------------------ @@ -85,6 +91,7 @@ connector_config() -> <<"enable">> => true, <<"description">> => <<"my connector">>, <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, <<"server">> => <<"127.0.0.1:1883">>, <<"resource_opts">> => #{ <<"health_check_interval">> => <<"15s">>, @@ -105,13 +112,6 @@ source_config(Overrides0) -> #{ <<"topic">> => <<"remote/topic">>, <<"qos">> => 2 - }, - <<"local">> => - #{ - <<"topic">> => <<"local/topic">>, - <<"qos">> => 2, - <<"retain">> => false, - <<"payload">> => <<"${payload}">> } }, <<"resource_opts">> => #{ @@ -134,6 +134,15 @@ source_config(Overrides0) -> replace(Key, Value, Proplist) -> lists:keyreplace(Key, 1, Proplist, {Key, Value}). +bridge_id(Config) -> + Type = ?config(source_type, Config), + Name = ?config(source_name, Config), + emqx_bridge_resource:bridge_id(Type, Name). + +hookpoint(Config) -> + BridgeId = bridge_id(Config), + emqx_bridge_resource:bridge_hookpoint(BridgeId). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -151,6 +160,11 @@ t_create_via_http(Config) -> ]}}, emqx_bridge_v2_testlib:list_bridges_http_api_v1() ), + ?assertMatch( + {ok, {{_, 200, _}, _, [#{<<"enable">> := true}]}}, + emqx_bridge_v2_testlib:list_connectors_http_api() + ), + NewSourceName = <<"my_other_source">>, {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_kind_api( @@ -173,3 +187,71 @@ t_create_via_http(Config) -> t_start_stop(Config) -> ok = emqx_bridge_v2_testlib:t_start_stop(Config, mqtt_connector_stopped), ok. + +t_receive_via_rule(Config) -> + SourceConfig = ?config(source_config, Config), + ?check_trace( + begin + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_connector_api(Config), + {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_kind_api(Config), + Hookpoint = hookpoint(Config), + RepublishTopic = <<"rep/t">>, + RemoteTopic = emqx_utils_maps:deep_get( + [<<"parameters">>, <<"remote">>, <<"topic">>], + SourceConfig + ), + RuleOpts = #{ + sql => <<"select * from \"", Hookpoint/binary, "\"">>, + actions => [ + %% #{function => console}, + #{ + function => republish, + args => #{ + topic => RepublishTopic, + payload => <<"${.}">>, + qos => 0, + retain => false, + user_properties => <<"${.pub_props.'User-Property'}">> + } + } + ] + }, + {ok, {{_, 201, _}, _, #{<<"id">> := RuleId}}} = + emqx_bridge_v2_testlib:create_rule_api(RuleOpts), + on_exit(fun() -> emqx_rule_engine:delete_rule(RuleId) end), + {ok, Client} = emqtt:start_link([{proto_ver, v5}]), + {ok, _} = emqtt:connect(Client), + {ok, _, [?RC_GRANTED_QOS_0]} = emqtt:subscribe(Client, RepublishTopic), + ok = emqtt:publish( + Client, + RemoteTopic, + #{'User-Property' => [{<<"key">>, <<"value">>}]}, + <<"mypayload">>, + _Opts = [] + ), + {publish, Msg} = + ?assertReceive( + {publish, #{ + topic := RepublishTopic, + retain := false, + qos := 0, + properties := #{'User-Property' := [{<<"key">>, <<"value">>}]} + }} + ), + Payload = emqx_utils_json:decode(maps:get(payload, Msg), [return_maps]), + ?assertMatch( + #{ + <<"event">> := Hookpoint, + <<"payload">> := <<"mypayload">> + }, + Payload + ), + emqtt:stop(Client), + ok + end, + fun(Trace) -> + ?assertEqual([], ?of_kind("action_references_nonexistent_bridges", Trace)), + ok + end + ), + ok. diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 83d387cd7..aae913001 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -326,7 +326,7 @@ schema("/connectors_probe") -> create_connector(ConnectorType, ConnectorName, Conf) end; '/connectors'(get, _Params) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), NodeReplies = emqx_connector_proto_v1:list_connectors_on_nodes(Nodes), case is_ok(NodeReplies) of {ok, NodeConnectors} -> @@ -674,7 +674,10 @@ unpack_connector_conf(Type, PackedConf) -> RawConf. format_action(ActionId) -> - element(2, emqx_bridge_v2:parse_id(ActionId)). + case emqx_bridge_v2:parse_id(ActionId) of + #{name := Name} -> + Name + end. is_ok(ok) -> ok; diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index cd8d597de..30e60df2b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -241,6 +241,12 @@ parse_user_properties(<<"${pub_props.'User-Property'}">>) -> %% we do not want to force users to select the value %% the value will be taken from Env.pub_props directly ?ORIGINAL_USER_PROPERTIES; +parse_user_properties(<<"${.pub_props.'User-Property'}">>) -> + %% keep the original + %% avoid processing this special variable because + %% we do not want to force users to select the value + %% the value will be taken from Env.pub_props directly + ?ORIGINAL_USER_PROPERTIES; parse_user_properties(<<"${", _/binary>> = V) -> %% use a variable emqx_template:parse(V); diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 70a7fc32c..b58d877e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -23,6 +23,7 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("stdlib/include/qlc.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([start_link/0]). @@ -482,8 +483,7 @@ with_parsed_rule(Params = #{id := RuleId, sql := Sql, actions := Actions}, Creat ok -> ok; {error, NonExistentBridgeIDs} -> - ?SLOG(error, #{ - msg => "action_references_nonexistent_bridges", + ?tp(error, "action_references_nonexistent_bridges", #{ rule_id => RuleId, nonexistent_bridge_ids => NonExistentBridgeIDs, hint => "this rule will be disabled" @@ -626,7 +626,7 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul {Type, Name} = emqx_bridge_resource:parse_bridge_id(BridgeID, #{atom_name => false}), case emqx_action_info:is_action_type(Type) of - true -> {action, Type, Name}; + true -> {source, Type, Name}; false -> {bridge_v1, Type, Name} end end, @@ -646,7 +646,8 @@ validate_bridge_existence_in_actions(#{actions := Actions, from := Froms} = _Rul fun({Kind, Type, Name}) -> LookupFn = case Kind of - action -> fun emqx_bridge_v2:lookup/2; + action -> fun emqx_bridge_v2:lookup_action/2; + source -> fun emqx_bridge_v2:lookup_source/2; bridge_v1 -> fun emqx_bridge:lookup/2 end, try From 8f304d3456688cd2b7d73703898b859ea178d42b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 16:24:52 -0300 Subject: [PATCH 44/62] test(bridge_v2_api): refactor suite to use CT matrix --- apps/emqx/test/emqx_common_test_helpers.erl | 29 +- .../test/emqx_bridge_v2_api_SUITE.erl | 1230 +++++++++-------- 2 files changed, 669 insertions(+), 590 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d9c9470eb..9438d227e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1389,29 +1389,40 @@ matrix_to_groups(Module, Cases) -> Cases ). -add_case_matrix(Module, Case, Acc0) -> - {RootGroup, Matrix} = Module:Case(matrix), +add_case_matrix(Module, TestCase, Acc0) -> + {MaybeRootGroup, Matrix} = + case Module:TestCase(matrix) of + {RootGroup0, Matrix0} -> + {RootGroup0, Matrix0}; + Matrix0 -> + {undefined, Matrix0} + end, lists:foldr( fun(Row, Acc) -> - add_group([RootGroup | Row], Acc, Case) + case MaybeRootGroup of + undefined -> + add_group(Row, Acc, TestCase); + RootGroup -> + add_group([RootGroup | Row], Acc, TestCase) + end end, Acc0, Matrix ). -add_group([], Acc, Case) -> - case lists:member(Case, Acc) of +add_group([], Acc, TestCase) -> + case lists:member(TestCase, Acc) of true -> Acc; false -> - [Case | Acc] + [TestCase | Acc] end; -add_group([Name | More], Acc, Cases) -> +add_group([Name | More], Acc, TestCases) -> case lists:keyfind(Name, 1, Acc) of false -> - [{Name, [], add_group(More, [], Cases)} | Acc]; + [{Name, [], add_group(More, [], TestCases)} | Acc]; {Name, [], SubGroup} -> - New = {Name, [], add_group(More, SubGroup, Cases)}, + New = {Name, [], add_group(More, SubGroup, TestCases)}, lists:keystore(Name, 1, Acc, New) end. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 0c34610ea..0ce8c620e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -24,9 +24,9 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/test_macros.hrl"). --define(ROOT, "actions"). +-define(ACTIONS_ROOT, "actions"). --define(CONNECTOR_NAME, <<"my_connector">>). +-define(ACTION_CONNECTOR_NAME, <<"my_connector">>). -define(RESOURCE(NAME, TYPE), #{ <<"enable">> => true, @@ -35,10 +35,10 @@ <<"name">> => NAME }). --define(CONNECTOR_TYPE_STR, "kafka_producer"). --define(CONNECTOR_TYPE, <>). +-define(ACTION_CONNECTOR_TYPE_STR, "kafka_producer"). +-define(ACTION_CONNECTOR_TYPE, <>). -define(KAFKA_BOOTSTRAP_HOST, <<"127.0.0.1:9092">>). --define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?CONNECTOR_TYPE)#{ +-define(KAFKA_CONNECTOR(Name, BootstrapHosts), ?RESOURCE(Name, ?ACTION_CONNECTOR_TYPE)#{ <<"authentication">> => <<"none">>, <<"bootstrap_hosts">> => BootstrapHosts, <<"connect_timeout">> => <<"5s">>, @@ -53,14 +53,14 @@ } }). --define(CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). --define(CONNECTOR, ?CONNECTOR(?CONNECTOR_NAME)). +-define(ACTIONS_CONNECTOR(Name), ?KAFKA_CONNECTOR(Name, ?KAFKA_BOOTSTRAP_HOST)). +-define(ACTIONS_CONNECTOR, ?ACTIONS_CONNECTOR(?ACTION_CONNECTOR_NAME)). -define(MQTT_LOCAL_TOPIC, <<"mqtt/local/topic">>). -define(BRIDGE_NAME, (atom_to_binary(?FUNCTION_NAME))). --define(BRIDGE_TYPE_STR, "kafka_producer"). --define(BRIDGE_TYPE, <>). --define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?BRIDGE_TYPE)#{ +-define(ACTION_TYPE_STR, "kafka_producer"). +-define(ACTION_TYPE, <>). +-define(KAFKA_BRIDGE(Name, Connector), ?RESOURCE(Name, ?ACTION_TYPE)#{ <<"connector">> => Connector, <<"kafka">> => #{ <<"buffer">> => #{ @@ -99,12 +99,12 @@ <<"health_check_interval">> => <<"32s">> } }). --define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE(Name), ?KAFKA_BRIDGE(Name, ?ACTION_CONNECTOR_NAME)). -define(KAFKA_BRIDGE_UPDATE(Name, Connector), maps:without([<<"name">>, <<"type">>], ?KAFKA_BRIDGE(Name, Connector)) ). --define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?CONNECTOR_NAME)). +-define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?ACTION_CONNECTOR_NAME)). -define(APPSPECS, [ emqx_conf, @@ -120,34 +120,27 @@ {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ). +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + -if(?EMQX_RELEASE_EDITION == ee). %% For now we got only kafka implementing `bridge_v2` and that is enterprise only. all() -> - [ - {group, single}, - {group, cluster_later_join}, - {group, cluster} - ]. + All0 = emqx_common_test_helpers:all(?MODULE), + All = All0 -- matrix_cases(), + Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()), + Groups ++ All. -else. all() -> []. -endif. +matrix_cases() -> + emqx_common_test_helpers:all(?MODULE). + groups() -> - AllTCs = emqx_common_test_helpers:all(?MODULE), - SingleOnlyTests = [ - t_bridges_probe, - t_broken_bridge_config, - t_fix_broken_bridge_config - ], - ClusterLaterJoinOnlyTCs = [ - t_cluster_later_join_metrics - ], - [ - {single, [], AllTCs -- ClusterLaterJoinOnlyTCs}, - {cluster_later_join, [], ClusterLaterJoinOnlyTCs}, - {cluster, [], (AllTCs -- SingleOnlyTests) -- ClusterLaterJoinOnlyTCs} - ]. + emqx_common_test_helpers:matrix_to_groups(?MODULE, matrix_cases()). suite() -> [{timetrap, {seconds, 60}}]. @@ -164,10 +157,12 @@ init_per_group(cluster = Name, Config) -> init_per_group(cluster_later_join = Name, Config) -> Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); -init_per_group(Name, Config) -> - WorkDir = filename:join(?config(priv_dir, Config), Name), +init_per_group(single = Group, Config) -> + WorkDir = filename:join(?config(priv_dir, Config), Group), Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), - init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]). + init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]); +init_per_group(actions, Config) -> + [{bridge_kind, action} | Config]. init_api(Config) -> Node = ?config(node, Config), @@ -193,8 +188,10 @@ end_per_group(Group, Config) when Group =:= cluster_later_join -> ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)); -end_per_group(_, Config) -> +end_per_group(single, Config) -> emqx_cth_suite:stop(?config(group_apps, Config)), + ok; +end_per_group(_Group, _Config) -> ok. init_per_testcase(t_action_types, Config) -> @@ -212,7 +209,7 @@ init_per_testcase(_TestCase, Config) -> Nodes -> [erpc:call(Node, ?MODULE, init_mocks, []) || Node <- Nodes] end, - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR, Config), + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config), Config. end_per_testcase(_TestCase, Config) -> @@ -227,6 +224,10 @@ end_per_testcase(_TestCase, Config) -> ok = emqx_common_test_helpers:call_janitor(), ok. +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + -define(CONNECTOR_IMPL, emqx_bridge_v2_dummy_connector). init_mocks() -> case emqx_release:edition() of @@ -243,7 +244,7 @@ init_mocks() -> ?CONNECTOR_IMPL, on_start, fun - (<<"connector:", ?CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> + (<<"connector:", ?ACTION_CONNECTOR_TYPE_STR, ":bad_", _/binary>>, _C) -> {ok, bad_connector_state}; (_I, _C) -> {ok, connector_state} @@ -280,442 +281,6 @@ clear_resources() -> emqx_connector:list() ). -%%------------------------------------------------------------------------------ -%% Testcases -%%------------------------------------------------------------------------------ - -%% We have to pretend testing a kafka bridge since at this point that's the -%% only one that's implemented. - -t_bridges_lifecycle(Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - {ok, 404, _} = request(get, uri([?ROOT, "foo"]), Config), - {ok, 404, _} = request(get, uri([?ROOT, "kafka_producer:foo"]), Config), - - %% need a var for patterns below - BridgeName = ?BRIDGE_NAME, - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _], - <<"connector">> := ?CONNECTOR_NAME, - <<"parameters">> := #{}, - <<"local_topic">> := _, - <<"resource_opts">> := _ - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME), - Config - ) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% list all bridges, assert bridge is in it - ?assertMatch( - {ok, 200, [ - #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - %% get the bridge by id - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ?assertMatch( - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := _ - }}, - request_json(post, uri([?ROOT, BridgeID, "brababbel"]), Config) - ), - - %% update bridge config - {ok, 201, _} = request(post, uri(["connectors"]), ?CONNECTOR(<<"foobla">>), Config), - ?assertMatch( - {ok, 200, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := BridgeName, - <<"connector">> := <<"foobla">>, - <<"enable">> := true, - <<"status">> := _, - <<"node_status">> := [_ | _] - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), - Config - ) - ), - - %% update bridge with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message1 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message1) - ), - - %% update bridge with connector of wrong type - {ok, 201, _} = - request( - post, - uri(["connectors"]), - (?CONNECTOR(<<"foobla2">>))#{ - <<"type">> => <<"azure_event_hub_producer">>, - <<"authentication">> => #{ - <<"username">> => <<"emqxuser">>, - <<"password">> => <<"topSecret">>, - <<"mechanism">> => <<"plain">> - }, - <<"ssl">> => #{ - <<"enable">> => true, - <<"server_name_indication">> => <<"auto">>, - <<"verify">> => <<"verify_none">>, - <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] - } - }, - Config - ), - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message2 - }} = - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message2) - ), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% try create with unknown connector name - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message3 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message3) - ), - - %% try create bridge with connector of wrong type - {ok, 400, #{ - <<"code">> := <<"BAD_REQUEST">>, - <<"message">> := Message4 - }} = - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), - Config - ), - ?assertMatch( - #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, - emqx_utils_json:decode(Message4) - ), - - %% make sure nothing has been created above - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% update a deleted bridge returns an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), - Config - ) - ), - - %% deleting a non-existing bridge should result in an error - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := _ - }}, - request_json(delete, uri([?ROOT, BridgeID]), Config) - ), - - %% try delete unknown bridge id - ?assertMatch( - {ok, 404, #{ - <<"code">> := <<"NOT_FOUND">>, - <<"message">> := <<"Invalid bridge ID", _/binary>> - }}, - request_json(delete, uri([?ROOT, "foo"]), Config) - ), - - %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), - {ok, 400, _} = request(post, uri([?ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), - ok. - -t_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - ?assertEqual( - {ok, 204, <<>>}, - request(delete, uri([?ROOT, BridgeID]), Config) - ), - - ?assertEqual( - {ok, 200, []}, - request_json(get, uri([?ROOT]), Config) - ), - - ok. - -t_fix_broken_bridge_config(Config) -> - emqx_cth_suite:stop_apps([emqx_bridge]), - BridgeName = ?BRIDGE_NAME, - StartOps = - #{ - config => - "actions {\n" - " " - ?BRIDGE_TYPE_STR - " {\n" - " " ++ binary_to_list(BridgeName) ++ - " {\n" - " connector = does_not_exist\n" - " enable = true\n" - " kafka {\n" - " topic = test-topic-one-partition\n" - " }\n" - " local_topic = \"mqtt/local/topic\"\n" - " resource_opts {health_check_interval = 32s}\n" - " }\n" - " }\n" - "}\n" - "\n", - schema_mod => emqx_bridge_v2_schema - }, - emqx_cth_suite:start_app(emqx_bridge, StartOps), - - ?assertMatch( - {ok, 200, [ - #{ - <<"name">> := BridgeName, - <<"type">> := ?BRIDGE_TYPE, - <<"connector">> := <<"does_not_exist">>, - <<"status">> := <<"disconnected">>, - <<"error">> := <<"Not installed">> - } - ]}, - request_json(get, uri([?ROOT]), Config) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), - request_json( - put, - uri([?ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?CONNECTOR_NAME), - Config - ), - - ?assertMatch( - {ok, 200, #{ - <<"connector">> := ?CONNECTOR_NAME, - <<"status">> := <<"connected">> - }}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - ok. - -t_start_bridge_unknown_node(Config) -> - {ok, 404, _} = - request( - post, - uri(["nodes", "thisbetterbenotanatomyet", ?ROOT, "kafka_producer:foo", start]), - Config - ), - {ok, 404, _} = - request( - post, - uri(["nodes", "undefined", ?ROOT, "kafka_producer:foo", start]), - Config - ). - -t_start_bridge_node(Config) -> - do_start_bridge(node, Config). - -t_start_bridge_cluster(Config) -> - do_start_bridge(cluster, Config). - -do_start_bridge(TestType, Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - Name = atom_to_binary(TestType), - ?assertMatch( - {ok, 201, #{ - <<"type">> := ?BRIDGE_TYPE, - <<"name">> := Name, - <<"enable">> := true, - <<"status">> := <<"connected">>, - <<"node_status">> := [_ | _] - }}, - request_json( - post, - uri([?ROOT]), - ?KAFKA_BRIDGE(Name), - Config - ) - ), - - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), - - %% start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - %% start a started bridge - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - ?assertMatch( - {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ROOT, BridgeID]), Config) - ), - - {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), - - %% Make start bridge fail - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, - Config - ), - - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, stop), - connector_operation(Config, ?BRIDGE_TYPE, ?CONNECTOR_NAME, start), - - {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% Make start bridge succeed - - expect_on_all_nodes( - ?CONNECTOR_IMPL, - on_add_channel, - fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, - Config - ), - - %% try to start again - {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), - - %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), - - %% Fail parse-id check - {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), - %% Looks ok but doesn't exist - {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), - ok. - expect_on_all_nodes(Mod, Function, Fun, Config) -> case ?config(cluster_nodes, Config) of undefined -> @@ -751,6 +316,548 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> ok = emqx_connector_resource:OperationName(ConnectorType, ConnectorName) end. +listen_on_random_port() -> + SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], + case gen_tcp:listen(0, SockOpts) of + {ok, Sock} -> + {ok, Port} = inet:port(Sock), + {Port, Sock}; + {error, Reason} when Reason /= eaddrinuse -> + {error, Reason} + end. + +request(Method, URL, Config) -> + request(Method, URL, [], Config). + +request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> + URL = operation_path(Type, Op, BridgeID, Config), + request(Method, URL, Body, Config); +request(Method, URL, Body, Config) -> + AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). + +request(Method, URL, Body, Decoder, Config) -> + case request(Method, URL, Body, Config) of + {ok, Code, Response} -> + case Decoder(Response) of + {error, _} = Error -> Error; + Decoded -> {ok, Code, Decoded} + end; + Otherwise -> + Otherwise + end. + +request_json(Method, URLLike, Config) -> + request(Method, URLLike, [], fun json/1, Config). + +request_json(Method, URLLike, Body, Config) -> + request(Method, URLLike, Body, fun json/1, Config). + +operation_path(node, Oper, BridgeID, Config) -> + uri(["nodes", ?config(node, Config), ?ACTIONS_ROOT, BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, _Config) -> + uri([?ACTIONS_ROOT, BridgeID, Oper]). + +enable_path(Enable, BridgeID) -> + uri([?ACTIONS_ROOT, BridgeID, "enable", Enable]). + +publish_message(Topic, Body, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). + +update_config(Path, Value, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, update_config, [Path, Value]). + +get_raw_config(Path, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx, get_raw_config, [Path]). + +add_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). + +delete_user_auth(Chain, AuthenticatorID, User, Config) -> + Node = ?config(node, Config), + erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). + +str(S) when is_list(S) -> S; +str(S) when is_binary(S) -> binary_to_list(S). + +json(B) when is_binary(B) -> + case emqx_utils_json:safe_decode(B, [return_maps]) of + {ok, Term} -> + Term; + {error, Reason} = Error -> + ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), + Error + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +%% We have to pretend testing a kafka bridge since at this point that's the +%% only one that's implemented. + +t_bridges_lifecycle(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_bridges_lifecycle(Config) -> + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "foo"]), Config), + {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "kafka_producer:foo"]), Config), + + %% need a var for patterns below + BridgeName = ?BRIDGE_NAME, + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _], + <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"parameters">> := #{}, + <<"local_topic">> := _, + <<"resource_opts">> := _ + }}, + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME), + Config + ) + ), + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + %% list all bridges, assert bridge is in it + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + %% get the bridge by id + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + ?assertMatch( + {ok, 200, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := _ + }}, + request_json(post, uri([?ACTIONS_ROOT, BridgeID, "brababbel"]), Config) + ), + + %% update bridge config + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR(<<"foobla">>), Config), + ?assertMatch( + {ok, 200, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := BridgeName, + <<"connector">> := <<"foobla">>, + <<"enable">> := true, + <<"status">> := _, + <<"node_status">> := [_ | _] + }}, + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), + Config + ) + ), + + %% update bridge with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message1 + }} = + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message1) + ), + + %% update bridge with connector of wrong type + {ok, 201, _} = + request( + post, + uri(["connectors"]), + (?ACTIONS_CONNECTOR(<<"foobla2">>))#{ + <<"type">> => <<"azure_event_hub_producer">>, + <<"authentication">> => #{ + <<"username">> => <<"emqxuser">>, + <<"password">> => <<"topSecret">>, + <<"mechanism">> => <<"plain">> + }, + <<"ssl">> => #{ + <<"enable">> => true, + <<"server_name_indication">> => <<"auto">>, + <<"verify">> => <<"verify_none">>, + <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] + } + }, + Config + ), + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message2 + }} = + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message2) + ), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% try create with unknown connector name + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message3 + }} = + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message3) + ), + + %% try create bridge with connector of wrong type + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := Message4 + }} = + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), + Config + ), + ?assertMatch( + #{<<"reason">> := <<"connector_not_found_or_wrong_type">>}, + emqx_utils_json:decode(Message4) + ), + + %% make sure nothing has been created above + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% update a deleted bridge returns an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), + Config + ) + ), + + %% deleting a non-existing bridge should result in an error + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := _ + }}, + request_json(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + %% try delete unknown bridge id + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Invalid bridge ID", _/binary>> + }}, + request_json(delete, uri([?ACTIONS_ROOT, "foo"]), Config) + ), + + %% Try create bridge with bad characters as name + {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), + {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), + ok. + +t_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + ?assertEqual( + {ok, 204, <<>>}, + request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ?assertEqual( + {ok, 200, []}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + ok. + +t_fix_broken_bridge_config(matrix) -> + [ + [single, actions] + ]; +t_fix_broken_bridge_config(Config) -> + emqx_cth_suite:stop_apps([emqx_bridge]), + BridgeName = ?BRIDGE_NAME, + StartOps = + #{ + config => + "actions {\n" + " " + ?ACTION_TYPE_STR + " {\n" + " " ++ binary_to_list(BridgeName) ++ + " {\n" + " connector = does_not_exist\n" + " enable = true\n" + " kafka {\n" + " topic = test-topic-one-partition\n" + " }\n" + " local_topic = \"mqtt/local/topic\"\n" + " resource_opts {health_check_interval = 32s}\n" + " }\n" + " }\n" + "}\n" + "\n", + schema_mod => emqx_bridge_v2_schema + }, + emqx_cth_suite:start_app(emqx_bridge, StartOps), + + ?assertMatch( + {ok, 200, [ + #{ + <<"name">> := BridgeName, + <<"type">> := ?ACTION_TYPE, + <<"connector">> := <<"does_not_exist">>, + <<"status">> := <<"disconnected">>, + <<"error">> := <<"Not installed">> + } + ]}, + request_json(get, uri([?ACTIONS_ROOT]), Config) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + request_json( + put, + uri([?ACTIONS_ROOT, BridgeID]), + ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, ?ACTION_CONNECTOR_NAME), + Config + ), + + ?assertMatch( + {ok, 200, #{ + <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"status">> := <<"connected">> + }}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + ok. + +t_start_bridge_unknown_node(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_unknown_node(Config) -> + {ok, 404, _} = + request( + post, + uri(["nodes", "thisbetterbenotanatomyet", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ), + {ok, 404, _} = + request( + post, + uri(["nodes", "undefined", ?ACTIONS_ROOT, "kafka_producer:foo", start]), + Config + ). + +t_start_bridge_node(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_node(Config) -> + do_start_bridge(node, Config). + +t_start_bridge_cluster(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; +t_start_bridge_cluster(Config) -> + do_start_bridge(cluster, Config). + +do_start_bridge(TestType, Config) -> + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + Name = atom_to_binary(TestType), + ?assertMatch( + {ok, 201, #{ + <<"type">> := ?ACTION_TYPE, + <<"name">> := Name, + <<"enable">> := true, + <<"status">> := <<"connected">>, + <<"node_status">> := [_ | _] + }}, + request_json( + post, + uri([?ACTIONS_ROOT]), + ?KAFKA_BRIDGE(Name), + Config + ) + ), + + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), + + %% start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + %% start a started bridge + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + ?assertMatch( + {ok, 200, #{<<"status">> := <<"connected">>}}, + request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + ), + + {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), + + %% Make start bridge fail + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {error, <<"my_error">>} end, + Config + ), + + connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, stop), + connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, start), + + {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% Make start bridge succeed + + expect_on_all_nodes( + ?CONNECTOR_IMPL, + on_add_channel, + fun(_, _, _ResId, _Channel) -> {ok, connector_state} end, + Config + ), + + %% try to start again + {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), + + %% delete the bridge + {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + + %% Fail parse-id check + {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), + %% Looks ok but doesn't exist + {ok, 404, _} = request(post, {operation, TestType, start, <<"webhook:cptn_hook">>}, Config), + ok. + %% t_start_stop_inconsistent_bridge_node(Config) -> %% start_stop_inconsistent_bridge(node, Config). @@ -861,6 +968,10 @@ connector_operation(Config, ConnectorType, ConnectorName, OperationName) -> %% {ok, 204, <<>>} = request(delete, uri([?ROOT, BridgeID]), Config), %% {ok, 200, []} = request_json(get, uri([?ROOT]), Config). +t_bridges_probe(matrix) -> + [ + [single, actions] + ]; t_bridges_probe(Config) -> {ok, 204, <<>>} = request( post, @@ -905,15 +1016,20 @@ t_bridges_probe(Config) -> ), ok. +t_cascade_delete_actions(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_cascade_delete_actions(Config) -> %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% then we add a a bridge, using POST %% POST /actions/ will create a bridge - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -931,10 +1047,10 @@ t_cascade_delete_actions(Config) -> %% delete the bridge will also delete the actions from the rules {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertMatch( {ok, 200, #{<<"actions">> := []}}, request_json(get, uri(["rules", RuleId]), Config) @@ -943,7 +1059,7 @@ t_cascade_delete_actions(Config) -> {ok, 201, _} = request( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ), @@ -960,19 +1076,24 @@ t_cascade_delete_actions(Config) -> ), {ok, 400, Body} = request( delete, - uri([?ROOT, BridgeID]), + uri([?ACTIONS_ROOT, BridgeID]), Config ), ?assertMatch(#{<<"rules">> := [_ | _]}, emqx_utils_json:decode(Body, [return_maps])), - {ok, 200, [_]} = request_json(get, uri([?ROOT]), Config), + {ok, 200, [_]} = request_json(get, uri([?ACTIONS_ROOT]), Config), %% Cleanup {ok, 204, _} = request( delete, - uri([?ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", + uri([?ACTIONS_ROOT, BridgeID]) ++ "?also_delete_dep_actions=true", Config ), - {ok, 200, []} = request_json(get, uri([?ROOT]), Config). + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config). +t_action_types(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_action_types(Config) -> Res = request_json(get, uri(["action_types"]), Config), ?assertMatch({ok, 200, _}, Res), @@ -981,11 +1102,16 @@ t_action_types(Config) -> ?assert(lists:all(fun is_binary/1, Types), #{types => Types}), ok. +t_bad_name(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_bad_name(Config) -> Name = <<"_bad_name">>, Res = request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(Name), Config ), @@ -1001,31 +1127,36 @@ t_bad_name(Config) -> ), ok. +t_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_metrics(Config) -> - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 0}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), - {ok, 200, Bridge} = request_json(get, uri([?ROOT, ActionID]), Config), + {ok, 200, Bridge} = request_json(get, uri([?ACTIONS_ROOT, ActionID]), Config), ?assertNot(maps:is_key(<<"metrics">>, Bridge)), ?assertNot(maps:is_key(<<"node_metrics">>, Bridge)), @@ -1041,12 +1172,12 @@ t_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{<<"matched">> := 1}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), %% check for absence of metrics when listing all bridges - {ok, 200, Bridges} = request_json(get, uri([?ROOT]), Config), + {ok, 200, Bridges} = request_json(get, uri([?ACTIONS_ROOT]), Config), ?assertNotMatch( [ #{ @@ -1058,21 +1189,26 @@ t_metrics(Config) -> ), ok. +t_reset_metrics(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_reset_metrics(Config) -> %% assert there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ROOT]), Config), + {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), ActionName = ?BRIDGE_NAME, ?assertMatch( {ok, 201, _}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(?BRIDGE_NAME), Config ) ), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, ActionName), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ActionName), Body = <<"my msg">>, _ = publish_message(?MQTT_LOCAL_TOPIC, Body, Config), @@ -1084,11 +1220,11 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 1}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), - {ok, 204, <<>>} = request(put, uri([?ROOT, ActionID, "metrics", "reset"]), Config), + {ok, 204, <<>>} = request(put, uri([?ACTIONS_ROOT, ActionID, "metrics", "reset"]), Config), ?retry( _Sleep0 = 200, @@ -1098,28 +1234,34 @@ t_reset_metrics(Config) -> <<"metrics">> := #{<<"matched">> := 0}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ) ), ok. +t_cluster_later_join_metrics(matrix) -> + [ + [cluster_later_join, actions] + ]; t_cluster_later_join_metrics(Config) -> [PrimaryNode, OtherNode | _] = ?config(cluster_nodes, Config), Name = ?BRIDGE_NAME, ActionParams = ?KAFKA_BRIDGE(Name), - ActionID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name), + ActionID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), ?check_trace( begin %% Create a bridge on only one of the nodes. - ?assertMatch({ok, 201, _}, request_json(post, uri([?ROOT]), ActionParams, Config)), + ?assertMatch( + {ok, 201, _}, request_json(post, uri([?ACTIONS_ROOT]), ActionParams, Config) + ), %% Pre-condition. ?assertMatch( {ok, 200, #{ <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), %% Now join the other node join with the api node. ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]), @@ -1130,7 +1272,7 @@ t_cluster_later_join_metrics(Config) -> <<"metrics">> := #{<<"success">> := _}, <<"node_metrics">> := [#{<<"metrics">> := #{}}, #{<<"metrics">> := #{}} | _] }}, - request_json(get, uri([?ROOT, ActionID, "metrics"]), Config) + request_json(get, uri([?ACTIONS_ROOT, ActionID, "metrics"]), Config) ), ok end, @@ -1138,94 +1280,20 @@ t_cluster_later_join_metrics(Config) -> ), ok. +t_raw_config_response_defaults(matrix) -> + [ + [single, actions], + [cluster, actions] + ]; t_raw_config_response_defaults(Config) -> Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), ?assertMatch( {ok, 201, #{<<"enable">> := true}}, request_json( post, - uri([?ROOT]), + uri([?ACTIONS_ROOT]), Params, Config ) ), ok. - -%%% helpers -listen_on_random_port() -> - SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], - case gen_tcp:listen(0, SockOpts) of - {ok, Sock} -> - {ok, Port} = inet:port(Sock), - {Port, Sock}; - {error, Reason} when Reason /= eaddrinuse -> - {error, Reason} - end. - -request(Method, URL, Config) -> - request(Method, URL, [], Config). - -request(Method, {operation, Type, Op, BridgeID}, Body, Config) -> - URL = operation_path(Type, Op, BridgeID, Config), - request(Method, URL, Body, Config); -request(Method, URL, Body, Config) -> - AuthHeader = emqx_common_test_http:auth_header(?config(api_key, Config)), - Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, - emqx_mgmt_api_test_util:request_api(Method, URL, [], AuthHeader, Body, Opts). - -request(Method, URL, Body, Decoder, Config) -> - case request(Method, URL, Body, Config) of - {ok, Code, Response} -> - case Decoder(Response) of - {error, _} = Error -> Error; - Decoded -> {ok, Code, Decoded} - end; - Otherwise -> - Otherwise - end. - -request_json(Method, URLLike, Config) -> - request(Method, URLLike, [], fun json/1, Config). - -request_json(Method, URLLike, Body, Config) -> - request(Method, URLLike, Body, fun json/1, Config). - -operation_path(node, Oper, BridgeID, Config) -> - uri(["nodes", ?config(node, Config), ?ROOT, BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID, _Config) -> - uri([?ROOT, BridgeID, Oper]). - -enable_path(Enable, BridgeID) -> - uri([?ROOT, BridgeID, "enable", Enable]). - -publish_message(Topic, Body, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, publish, [emqx_message:make(Topic, Body)]). - -update_config(Path, Value, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, update_config, [Path, Value]). - -get_raw_config(Path, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx, get_raw_config, [Path]). - -add_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, add_user, [Chain, AuthenticatorID, User]). - -delete_user_auth(Chain, AuthenticatorID, User, Config) -> - Node = ?config(node, Config), - erpc:call(Node, emqx_authentication, delete_user, [Chain, AuthenticatorID, User]). - -str(S) when is_list(S) -> S; -str(S) when is_binary(S) -> binary_to_list(S). - -json(B) when is_binary(B) -> - case emqx_utils_json:safe_decode(B, [return_maps]) of - {ok, Term} -> - Term; - {error, Reason} = Error -> - ct:pal("Failed to decode json: ~p~n~p", [Reason, B]), - Error - end. From fc88a1ed1ee2b4818a0ca725c03aae3b177c1538 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 17:27:23 -0300 Subject: [PATCH 45/62] test(sources_api): add some tests to cover `/sources` HTTP API Also fixes a bug with `DELETE /sources/:id` --- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 2 +- .../test/emqx_bridge_v2_api_SUITE.erl | 277 ++++++++++++++---- 2 files changed, 221 insertions(+), 58 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index d4401cfd0..e8a500e85 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -786,7 +786,7 @@ handle_update(ConfRootKey, Id, Conf0) -> handle_delete(ConfRootKey, Id, QueryStringOpts) -> ?TRY_PARSE_ID( Id, - case emqx_bridge_v2:lookup(BridgeType, BridgeName) of + case emqx_bridge_v2:lookup(ConfRootKey, BridgeType, BridgeName) of {ok, _} -> AlsoDeleteActions = case maps:get(<<"also_delete_dep_actions">>, QueryStringOpts, <<"false">>) of diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 0ce8c620e..5ef897369 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -25,8 +25,10 @@ -include_lib("snabbkaffe/include/test_macros.hrl"). -define(ACTIONS_ROOT, "actions"). +-define(SOURCES_ROOT, "sources"). -define(ACTION_CONNECTOR_NAME, <<"my_connector">>). +-define(SOURCE_CONNECTOR_NAME, <<"my_connector">>). -define(RESOURCE(NAME, TYPE), #{ <<"enable">> => true, @@ -106,6 +108,9 @@ ). -define(KAFKA_BRIDGE_UPDATE(Name), ?KAFKA_BRIDGE_UPDATE(Name, ?ACTION_CONNECTOR_NAME)). +-define(SOURCE_TYPE_STR, "mqtt"). +-define(SOURCE_TYPE, <>). + -define(APPSPECS, [ emqx_conf, emqx, @@ -162,7 +167,11 @@ init_per_group(single = Group, Config) -> Apps = emqx_cth_suite:start(?APPSPECS ++ [?APPSPEC_DASHBOARD], #{work_dir => WorkDir}), init_api([{group, single}, {group_apps, Apps}, {node, node()} | Config]); init_per_group(actions, Config) -> - [{bridge_kind, action} | Config]. + [{bridge_kind, action} | Config]; +init_per_group(sources, Config) -> + [{bridge_kind, source} | Config]; +init_per_group(_Group, Config) -> + Config. init_api(Config) -> Node = ?config(node, Config), @@ -209,7 +218,17 @@ init_per_testcase(_TestCase, Config) -> Nodes -> [erpc:call(Node, ?MODULE, init_mocks, []) || Node <- Nodes] end, - {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config), + case ?config(bridge_kind, Config) of + action -> + {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR, Config); + source -> + {ok, 201, _} = request( + post, + uri(["connectors"]), + source_connector_create_config(#{}), + Config + ) + end, Config. end_per_testcase(_TestCase, Config) -> @@ -268,18 +287,7 @@ init_mocks() -> ok. clear_resources() -> - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_bridge_v2:remove(Type, Name) - end, - emqx_bridge_v2:list() - ), - lists:foreach( - fun(#{type := Type, name := Name}) -> - ok = emqx_connector:remove(Type, Name) - end, - emqx_connector:list() - ). + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(). expect_on_all_nodes(Mod, Function, Fun, Config) -> case ?config(cluster_nodes, Config) of @@ -394,6 +402,135 @@ json(B) when is_binary(B) -> Error end. +group_path(Config) -> + case emqx_common_test_helpers:group_path(Config) of + [] -> + undefined; + Path -> + Path + end. + +source_connector_config_base() -> + #{ + <<"enable">> => true, + <<"description">> => <<"my connector">>, + <<"pool_size">> => 3, + <<"proto_ver">> => <<"v5">>, + <<"server">> => <<"127.0.0.1:1883">>, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"15s">>, + <<"start_after_created">> => true, + <<"start_timeout">> => <<"5s">> + } + }. + +source_connector_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_connector_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE, + <<"name">> => ?SOURCE_CONNECTOR_NAME + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_config_base() -> + #{ + <<"enable">> => true, + <<"connector">> => ?SOURCE_CONNECTOR_NAME, + <<"parameters">> => + #{ + <<"remote">> => + #{ + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 + } + }, + <<"resource_opts">> => #{ + <<"batch_size">> => 1, + <<"batch_time">> => <<"0ms">>, + <<"buffer_mode">> => <<"memory_only">>, + <<"buffer_seg_bytes">> => <<"10MB">>, + <<"health_check_interval">> => <<"15s">>, + <<"inflight_window">> => 100, + <<"max_buffer_bytes">> => <<"256MB">>, + <<"metrics_flush_interval">> => <<"1s">>, + <<"query_mode">> => <<"sync">>, + <<"request_ttl">> => <<"45s">>, + <<"resume_interval">> => <<"15s">>, + <<"worker_pool_size">> => <<"1">> + } + }. + +source_create_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + Conf0 = maps:merge( + source_config_base(), + #{ + <<"enable">> => true, + <<"type">> => ?SOURCE_TYPE + } + ), + maps:merge( + Conf0, + Overrides + ). + +source_update_config(Overrides0) -> + Overrides = emqx_utils_maps:binary_key_map(Overrides0), + maps:merge( + source_config_base(), + Overrides + ). + +get_common_values(Kind, FnName) -> + case Kind of + actions -> + #{ + api_root_key => ?ACTIONS_ROOT, + type => ?ACTION_TYPE, + default_connector_name => ?ACTION_CONNECTOR_NAME, + create_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE(Name, ConnectorName) + end, + update_config_fn => + fun(Overrides) -> + Name = maps:get(name, Overrides, FnName), + ConnectorName = maps:get(connector, Overrides, ?ACTION_CONNECTOR_NAME), + ?KAFKA_BRIDGE_UPDATE(Name, ConnectorName) + end, + create_connector_config_fn => + fun(Overrides) -> + ConnectorName = maps:get(name, Overrides, ?ACTION_CONNECTOR_NAME), + ?ACTIONS_CONNECTOR(ConnectorName) + end + }; + sources -> + #{ + api_root_key => ?SOURCES_ROOT, + type => ?SOURCE_TYPE, + default_connector_name => ?SOURCE_CONNECTOR_NAME, + create_config_fn => fun(Overrides0) -> + Overrides = + case Overrides0 of + #{name := _} -> Overrides0; + _ -> Overrides0#{name => FnName} + end, + source_create_config(Overrides) + end, + update_config_fn => fun source_update_config/1, + create_connector_config_fn => fun source_connector_create_config/1 + } + end. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -404,76 +541,95 @@ json(B) when is_binary(B) -> t_bridges_lifecycle(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_bridges_lifecycle(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + FnName = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn, + update_config_fn := UpdateConfigFn, + create_connector_config_fn := CreateConnectorConfigFn + } = get_common_values(Kind, FnName), %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), - {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "foo"]), Config), - {ok, 404, _} = request(get, uri([?ACTIONS_ROOT, "kafka_producer:foo"]), Config), + {ok, 404, _} = request(get, uri([APIRootKey, "foo"]), Config), + {ok, 404, _} = request(get, uri([APIRootKey, "kafka_producer:foo"]), Config), %% need a var for patterns below - BridgeName = ?BRIDGE_NAME, + BridgeName = FnName, + CreateRes = request_json( + post, + uri([APIRootKey]), + CreateConfigFn(#{}), + Config + ), ?assertMatch( {ok, 201, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], - <<"connector">> := ?ACTION_CONNECTOR_NAME, + <<"connector">> := DefaultConnectorName, <<"parameters">> := #{}, - <<"local_topic">> := _, <<"resource_opts">> := _ }}, - request_json( - post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME), - Config - ) + CreateRes, + #{name => BridgeName, type => Type, connector => DefaultConnectorName} ), + case Kind of + actions -> + ?assertMatch({ok, 201, #{<<"local_topic">> := _}}, CreateRes); + sources -> + ok + end, %% list all bridges, assert bridge is in it ?assertMatch( {ok, 200, [ #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] } ]}, - request_json(get, uri([?ACTIONS_ROOT]), Config) + request_json(get, uri([APIRootKey]), Config) ), %% list all bridges, assert bridge is in it ?assertMatch( {ok, 200, [ #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] } ]}, - request_json(get, uri([?ACTIONS_ROOT]), Config) + request_json(get, uri([APIRootKey]), Config) ), %% get the bridge by id - BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, ?BRIDGE_NAME), + BridgeID = emqx_bridge_resource:bridge_id(Type, ?BRIDGE_NAME), ?assertMatch( {ok, 200, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"enable">> := true, <<"status">> := _, <<"node_status">> := [_ | _] }}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), ?assertMatch( @@ -481,14 +637,19 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := _ }}, - request_json(post, uri([?ACTIONS_ROOT, BridgeID, "brababbel"]), Config) + request_json(post, uri([APIRootKey, BridgeID, "brababbel"]), Config) ), %% update bridge config - {ok, 201, _} = request(post, uri(["connectors"]), ?ACTIONS_CONNECTOR(<<"foobla">>), Config), + {ok, 201, _} = request( + post, + uri(["connectors"]), + CreateConnectorConfigFn(#{name => <<"foobla">>}), + Config + ), ?assertMatch( {ok, 200, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := BridgeName, <<"connector">> := <<"foobla">>, <<"enable">> := true, @@ -497,8 +658,8 @@ t_bridges_lifecycle(Config) -> }}, request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla">>}), Config ) ), @@ -510,8 +671,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"does_not_exist">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"does_not_exist">>}), Config ), ?assertMatch( @@ -546,8 +707,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME, <<"foobla2">>), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{connector => <<"foobla2">>}), Config ), ?assertMatch( @@ -556,8 +717,8 @@ t_bridges_lifecycle(Config) -> ), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% try create with unknown connector name {ok, 400, #{ @@ -566,8 +727,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"does_not_exist">>), + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"does_not_exist">>}), Config ), ?assertMatch( @@ -582,8 +743,8 @@ t_bridges_lifecycle(Config) -> }} = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(?BRIDGE_NAME, <<"foobla2">>), + uri([APIRootKey]), + CreateConfigFn(#{connector => <<"foobla2">>}), Config ), ?assertMatch( @@ -592,7 +753,7 @@ t_bridges_lifecycle(Config) -> ), %% make sure nothing has been created above - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% update a deleted bridge returns an error ?assertMatch( @@ -602,8 +763,8 @@ t_bridges_lifecycle(Config) -> }}, request_json( put, - uri([?ACTIONS_ROOT, BridgeID]), - ?KAFKA_BRIDGE_UPDATE(?BRIDGE_NAME), + uri([APIRootKey, BridgeID]), + UpdateConfigFn(#{}), Config ) ), @@ -614,7 +775,7 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"NOT_FOUND">>, <<"message">> := _ }}, - request_json(delete, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(delete, uri([APIRootKey, BridgeID]), Config) ), %% try delete unknown bridge id @@ -623,12 +784,14 @@ t_bridges_lifecycle(Config) -> <<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Invalid bridge ID", _/binary>> }}, - request_json(delete, uri([?ACTIONS_ROOT, "foo"]), Config) + request_json(delete, uri([APIRootKey, "foo"]), Config) ), %% Try create bridge with bad characters as name - {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"隋达"/utf8>>), Config), - {ok, 400, _} = request(post, uri([?ACTIONS_ROOT]), ?KAFKA_BRIDGE(<<"a.b">>), Config), + {ok, 400, _} = request( + post, uri([APIRootKey]), CreateConfigFn(#{name => <<"隋达"/utf8>>}), Config + ), + {ok, 400, _} = request(post, uri([APIRootKey]), CreateConfigFn(#{name => <<"a.b">>}), Config), ok. t_broken_bridge_config(matrix) -> From 007af20a30472a1b22c5615c50c6f6d2b1a28d76 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 12 Jan 2024 17:59:11 -0300 Subject: [PATCH 46/62] test(bridge_v2_api): adapt more tests to sources --- .../test/emqx_bridge_v2_api_SUITE.erl | 80 +++++++++++++------ 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 5ef897369..d24d4feac 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -255,6 +255,8 @@ init_mocks() -> meck:expect(emqx_connector_ee_schema, resource_type, 1, ?CONNECTOR_IMPL), ok; ce -> + meck:new(emqx_connector_resource, [passthrough, no_link]), + meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), ok end, meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), @@ -363,9 +365,13 @@ request_json(Method, URLLike, Body, Config) -> request(Method, URLLike, Body, fun json/1, Config). operation_path(node, Oper, BridgeID, Config) -> - uri(["nodes", ?config(node, Config), ?ACTIONS_ROOT, BridgeID, Oper]); -operation_path(cluster, Oper, BridgeID, _Config) -> - uri([?ACTIONS_ROOT, BridgeID, Oper]). + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri(["nodes", ?config(node, Config), APIRootKey, BridgeID, Oper]); +operation_path(cluster, Oper, BridgeID, Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), + #{api_root_key := APIRootKey} = get_common_values(Kind, <<"unused">>), + uri([APIRootKey, BridgeID, Oper]). enable_path(Enable, BridgeID) -> uri([?ACTIONS_ROOT, BridgeID, "enable", Enable]). @@ -935,7 +941,9 @@ t_start_bridge_unknown_node(Config) -> t_start_bridge_node(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_start_bridge_node(Config) -> do_start_bridge(node, Config). @@ -943,19 +951,28 @@ t_start_bridge_node(Config) -> t_start_bridge_cluster(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_start_bridge_cluster(Config) -> do_start_bridge(cluster, Config). do_start_bridge(TestType, Config) -> - %% assert we there's no bridges at first - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), - + [_SingleOrCluster, Kind | _] = group_path(Config), Name = atom_to_binary(TestType), + #{ + api_root_key := APIRootKey, + type := Type, + default_connector_name := DefaultConnectorName, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + %% assert we there's no bridges at first + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), + ?assertMatch( {ok, 201, #{ - <<"type">> := ?ACTION_TYPE, + <<"type">> := Type, <<"name">> := Name, <<"enable">> := true, <<"status">> := <<"connected">>, @@ -963,25 +980,25 @@ do_start_bridge(TestType, Config) -> }}, request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(Name), + uri([APIRootKey]), + CreateConfigFn(#{name => Name}), Config ) ), - BridgeID = emqx_bridge_resource:bridge_id(?ACTION_TYPE, Name), + BridgeID = emqx_bridge_resource:bridge_id(Type, Name), %% start again {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), ?assertMatch( {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), %% start a started bridge {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), ?assertMatch( {ok, 200, #{<<"status">> := <<"connected">>}}, - request_json(get, uri([?ACTIONS_ROOT, BridgeID]), Config) + request_json(get, uri([APIRootKey, BridgeID]), Config) ), {ok, 400, _} = request(post, {operation, TestType, invalidop, BridgeID}, Config), @@ -994,8 +1011,8 @@ do_start_bridge(TestType, Config) -> Config ), - connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, stop), - connector_operation(Config, ?ACTION_TYPE, ?ACTION_CONNECTOR_NAME, start), + connector_operation(Config, Type, DefaultConnectorName, stop), + connector_operation(Config, Type, DefaultConnectorName, start), {ok, 400, _} = request(post, {operation, TestType, start, BridgeID}, Config), @@ -1012,8 +1029,8 @@ do_start_bridge(TestType, Config) -> {ok, 204, <<>>} = request(post, {operation, TestType, start, BridgeID}, Config), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri([?ACTIONS_ROOT, BridgeID]), Config), - {ok, 200, []} = request_json(get, uri([?ACTIONS_ROOT]), Config), + {ok, 204, <<>>} = request(delete, uri([APIRootKey, BridgeID]), Config), + {ok, 200, []} = request_json(get, uri([APIRootKey]), Config), %% Fail parse-id check {ok, 404, _} = request(post, {operation, TestType, start, <<"wreckbook_fugazi">>}, Config), @@ -1268,14 +1285,21 @@ t_action_types(Config) -> t_bad_name(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_bad_name(Config) -> + [_SingleOrCluster, Kind | _] = group_path(Config), Name = <<"_bad_name">>, + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), Res = request_json( post, - uri([?ACTIONS_ROOT]), - ?KAFKA_BRIDGE(Name), + uri([APIRootKey]), + CreateConfigFn(#{}), Config ), ?assertMatch({ok, 400, #{<<"message">> := _}}, Res), @@ -1446,15 +1470,23 @@ t_cluster_later_join_metrics(Config) -> t_raw_config_response_defaults(matrix) -> [ [single, actions], - [cluster, actions] + [single, sources], + [cluster, actions], + [cluster, sources] ]; t_raw_config_response_defaults(Config) -> - Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), + [_SingleOrCluster, Kind | _] = group_path(Config), + Name = atom_to_binary(?FUNCTION_NAME), + #{ + api_root_key := APIRootKey, + create_config_fn := CreateConfigFn + } = get_common_values(Kind, Name), + Params = maps:remove(<<"enable">>, CreateConfigFn(#{})), ?assertMatch( {ok, 201, #{<<"enable">> := true}}, request_json( post, - uri([?ACTIONS_ROOT]), + uri([APIRootKey]), Params, Config ) From 938429f3518359ac1ef148ca38a3d728bed8ddf0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jan 2024 13:55:11 -0300 Subject: [PATCH 47/62] chore(mqtt_bridge): change schema to remote `remote` sub-fields and hide `local` `local` is still needed for backwards compatibility --- .../test/emqx_bridge_v2_api_SUITE.erl | 7 +- .../src/emqx_bridge_mqtt_connector.erl | 28 +++++--- .../src/emqx_bridge_mqtt_ingress.erl | 10 +-- .../emqx_bridge_mqtt_pubsub_action_info.erl | 65 +++++++++++++------ .../src/emqx_bridge_mqtt_pubsub_schema.erl | 36 +++++++--- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 9 +-- 6 files changed, 101 insertions(+), 54 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index d24d4feac..fabaadb92 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -451,11 +451,8 @@ source_config_base() -> <<"connector">> => ?SOURCE_CONNECTOR_NAME, <<"parameters">> => #{ - <<"remote">> => - #{ - <<"topic">> => <<"remote/topic">>, - <<"qos">> => 2 - } + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 }, <<"resource_opts">> => #{ <<"batch_size">> => 1, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 18af6ee11..9aae73bd2 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -127,8 +127,9 @@ on_add_channel( true -> ok end, - ChannelState0 = maps:get(parameters, ChannelConfig), - ChannelState = emqx_bridge_mqtt_egress:config(ChannelState0), + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState = emqx_bridge_mqtt_egress:config(#{remote => RemoteParams, local => LocalParams}), NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels), NewState = OldState#{installed_channels => NewInstalledChannels}, {ok, NewState}; @@ -144,15 +145,18 @@ on_add_channel( #{hookpoints := HookPoints} = ChannelConfig ) -> %% Add ingress channel - ChannelState0 = maps:get(parameters, ChannelConfig), - ChannelState1 = ChannelState0#{ + RemoteParams0 = maps:get(parameters, ChannelConfig), + {LocalParams, RemoteParams} = take(local, RemoteParams0, #{}), + ChannelState0 = #{ hookpoints => HookPoints, server => Server, - config_root => sources + config_root => sources, + local => LocalParams, + remote => RemoteParams }, - ChannelState2 = mk_ingress_config(ChannelId, ChannelState1, TopicToHandlerIndex), - ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState2), - NewInstalledChannels = maps:put(ChannelId, ChannelState2, InstalledChannels), + ChannelState1 = mk_ingress_config(ChannelId, ChannelState0, TopicToHandlerIndex), + ok = emqx_bridge_mqtt_ingress:subscribe_channel(PoolName, ChannelState1), + NewInstalledChannels = maps:put(ChannelId, ChannelState1, InstalledChannels), NewState = OldState#{installed_channels => NewInstalledChannels}, {ok, NewState}. @@ -500,3 +504,11 @@ connect(Pid, Name) -> handle_disconnect(_Reason) -> ok. + +take(Key, Map0, Default) -> + case maps:take(Key, Map0) of + {Value, Map} -> + {Value, Map}; + error -> + {Default, Map0} + end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl index d59318a84..369238ecf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl @@ -178,11 +178,13 @@ config(#{ingress_list := IngressList} = Conf, Name, TopicToHandlerIndex) -> ], Conf#{ingress_list => NewIngressList}. -fix_remote_config(#{remote := RC, local := LC}, BridgeName, TopicToHandlerIndex, Conf) -> - FixedConf = Conf#{ - remote => parse_remote(RC, BridgeName), - local => emqx_bridge_mqtt_msg:parse(LC) +fix_remote_config(#{remote := RC}, BridgeName, TopicToHandlerIndex, Conf) -> + FixedConf0 = Conf#{ + remote => parse_remote(RC, BridgeName) }, + FixedConf = emqx_utils_maps:update_if_present( + local, fun emqx_bridge_mqtt_msg:parse/1, FixedConf0 + ), insert_to_topic_to_handler_index(FixedConf, TopicToHandlerIndex, BridgeName), FixedConf. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index e4a4fcd19..cf7a5bc04 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -95,8 +95,11 @@ bridge_v1_config_to_action_config_helper( LocalTopicMap = maps:get(<<"local">>, EgressMap0, #{}), LocalTopic = maps:get(<<"topic">>, LocalTopicMap, undefined), EgressMap1 = maps:without([<<"local">>, <<"pool_size">>], EgressMap0), + LocalParams = maps:get(<<"local">>, EgressMap0, #{}), + EgressMap2 = emqx_utils_maps:unindent(<<"remote">>, EgressMap1), + EgressMap = maps:put(<<"local">>, LocalParams, EgressMap2), %% Add parameters field (Egress map) to the action config - ConfigMap2 = maps:put(<<"parameters">>, EgressMap1, ConfigMap1), + ConfigMap2 = maps:put(<<"parameters">>, EgressMap, ConfigMap1), ConfigMap3 = case LocalTopic of undefined -> @@ -107,7 +110,7 @@ bridge_v1_config_to_action_config_helper( {action, mqtt, ConfigMap3}; bridge_v1_config_to_action_config_helper( #{ - <<"ingress">> := IngressMap + <<"ingress">> := IngressMap0 } = Config, ConnectorName ) -> @@ -117,9 +120,12 @@ bridge_v1_config_to_action_config_helper( ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), - IngressMap1 = maps:remove(<<"pool_size">>, IngressMap), + IngressMap1 = maps:without([<<"pool_size">>, <<"local">>], IngressMap0), + LocalParams = maps:get(<<"local">>, IngressMap0, #{}), + IngressMap2 = emqx_utils_maps:unindent(<<"remote">>, IngressMap1), + IngressMap = maps:put(<<"local">>, LocalParams, IngressMap2), %% Add parameters field (Egress map) to the action config - ConfigMap2 = maps:put(<<"parameters">>, IngressMap1, ConfigMap1), + ConfigMap2 = maps:put(<<"parameters">>, IngressMap, ConfigMap1), {source, mqtt, ConfigMap2}; bridge_v1_config_to_action_config_helper( _Config, @@ -182,7 +188,7 @@ check_and_simplify_bridge_v1_config(SimplifiedConfig) -> connector_action_config_to_bridge_v1_config( ConnectorConfig, ActionConfig ) -> - Params = maps:get(<<"parameters">>, ActionConfig, #{}), + Params0 = maps:get(<<"parameters">>, ActionConfig, #{}), ResourceOptsConnector = maps:get(<<"resource_opts">>, ConnectorConfig, #{}), ResourceOptsAction = maps:get(<<"resource_opts">>, ActionConfig, #{}), ResourceOpts0 = maps:merge(ResourceOptsConnector, ResourceOptsAction), @@ -194,37 +200,54 @@ connector_action_config_to_bridge_v1_config( ResourceOpts = maps:with(V1ResourceOptsFields, ResourceOpts0), %% Check the direction of the action Direction = - case maps:get(<<"remote">>, Params) of - #{<<"retain">> := _} -> - %% Only source has retain + case is_map_key(<<"retain">>, Params0) of + %% Only source has retain + true -> <<"publisher">>; - _ -> + false -> <<"subscriber">> end, - Parms2 = maps:remove(<<"direction">>, Params), + Params1 = maps:remove(<<"direction">>, Params0), + Params = maps:remove(<<"local">>, Params1), + %% hidden; for backwards compatibility + LocalParams = maps:get(<<"local">>, Params1, #{}), DefaultPoolSize = emqx_connector_schema_lib:pool_size(default), PoolSize = maps:get(<<"pool_size">>, ConnectorConfig, DefaultPoolSize), - Parms3 = maps:put(<<"pool_size">>, PoolSize, Parms2), ConnectorConfig2 = maps:remove(<<"pool_size">>, ConnectorConfig), LocalTopic = maps:get(<<"local_topic">>, ActionConfig, undefined), BridgeV1Conf0 = case {Direction, LocalTopic} of {<<"publisher">>, undefined} -> - #{<<"egress">> => Parms3}; + #{ + <<"egress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + }; {<<"publisher">>, LocalT} -> #{ <<"egress">> => - maps:merge( - Parms3, #{ - <<"local">> => - #{ - <<"topic">> => LocalT - } - } - ) + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => + maps:merge( + LocalParams, + #{<<"topic">> => LocalT} + ) + } }; {<<"subscriber">>, _} -> - #{<<"ingress">> => Parms3} + #{ + <<"ingress">> => + #{ + <<"pool_size">> => PoolSize, + <<"remote">> => Params, + <<"local">> => LocalParams + } + } end, BridgeV1Conf1 = maps:merge(BridgeV1Conf0, ConnectorConfig2), BridgeV1Conf2 = BridgeV1Conf1#{ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index b4c2b63ba..4cf092a60 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -56,10 +56,18 @@ fields("mqtt_publisher_action") -> ) ); fields(action_parameters) -> - Fields0 = emqx_bridge_mqtt_connector_schema:fields("egress"), - Fields1 = proplists:delete(pool_size, Fields0), - Fields2 = proplists:delete(local, Fields1), - Fields2; + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "egress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("egress_remote") + ]; fields(source) -> {mqtt, mk( @@ -71,8 +79,8 @@ fields(source) -> )}; fields("mqtt_subscriber_source") -> emqx_bridge_v2_schema:make_consumer_action_schema( - hoconsc:mk( - hoconsc:ref(?MODULE, ingress_parameters), + mk( + ref(?MODULE, ingress_parameters), #{ required => true, desc => ?DESC("source_parameters") @@ -80,10 +88,18 @@ fields("mqtt_subscriber_source") -> ) ); fields(ingress_parameters) -> - Fields0 = emqx_bridge_mqtt_connector_schema:fields("ingress"), - Fields1 = proplists:delete(pool_size, Fields0), - %% FIXME: should we make `local` hidden? - Fields1; + [ + %% for backwards compatibility + {local, + mk( + ref(emqx_bridge_mqtt_connector_schema, "ingress_local"), + #{ + default => #{}, + importance => ?IMPORTANCE_HIDDEN + } + )} + | emqx_bridge_mqtt_connector_schema:fields("ingress_remote") + ]; fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index 5569a826b..a0b3edfa7 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -108,11 +108,8 @@ source_config(Overrides0) -> <<"connector">> => <<"please override">>, <<"parameters">> => #{ - <<"remote">> => - #{ - <<"topic">> => <<"remote/topic">>, - <<"qos">> => 2 - } + <<"topic">> => <<"remote/topic">>, + <<"qos">> => 2 }, <<"resource_opts">> => #{ <<"batch_size">> => 1, @@ -197,7 +194,7 @@ t_receive_via_rule(Config) -> Hookpoint = hookpoint(Config), RepublishTopic = <<"rep/t">>, RemoteTopic = emqx_utils_maps:deep_get( - [<<"parameters">>, <<"remote">>, <<"topic">>], + [<<"parameters">>, <<"topic">>], SourceConfig ), RuleOpts = #{ From 440a543a85e27977d357cc1bd707088f80792080 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jan 2024 13:56:17 -0300 Subject: [PATCH 48/62] docs: fix typo --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 8f36fd700..262ada984 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,7 +49,7 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). -%% only for testting/mocking +%% only for testing/mocking -export([supported_versions/1]). -define(BPAPI_NAME, emqx_bridge). From c6cd3adccbca63697035dc36ac914da61c113484 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 16 Jan 2024 11:51:53 +0100 Subject: [PATCH 49/62] refactor: fix type upgrade calls and move compatiblitly logic Some bridge V1 to V2 calls were wrong but did not seem to cause issues (perhaps due to locking test coverage). This commit also move compatibility logic from the API module to the emqx_bridge_v2 module where most of the compatibility logic exists. --- apps/emqx_bridge/src/emqx_bridge.erl | 4 ++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 6 +----- apps/emqx_bridge/src/emqx_bridge_resource.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_v2.erl | 8 ++++++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 7df55c81c..e27748610 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -352,10 +352,10 @@ get_metrics(ActionType, Name) -> true -> case emqx_bridge_v2:bridge_v1_is_valid(ActionType, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(ActionType), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(ActionType), try ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( - ActionType, Name + BridgeV2Type, Name ), emqx_bridge_v2:get_metrics(ConfRootKey, BridgeV2Type, Name) catch diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 262ada984..7e929a233 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -554,11 +554,7 @@ schema("/bridges_probe") -> case emqx_bridge_v2:is_bridge_v2_type(BridgeType) of true -> try - ConfRootKey = emqx_bridge_v2:get_conf_root_key_if_only_one( - BridgeType, BridgeName - ), - BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeType), - ok = emqx_bridge_v2:reset_metrics(ConfRootKey, BridgeV2Type, BridgeName), + ok = emqx_bridge_v2:bridge_v1_reset_metrics(BridgeType, BridgeName), ?NO_CONTENT catch error:Reason -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ec7a7431b..143956b5d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -147,7 +147,7 @@ reset_metrics(ResourceId) -> true -> case emqx_bridge_v2:bridge_v1_is_valid(Type, Name) of true -> - BridgeV2Type = emqx_bridge_v2:bridge_v2_type_to_connector_type(Type), + BridgeV2Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(Type), emqx_bridge_v2:reset_metrics(BridgeV2Type, Name); false -> {error, not_bridge_v1_compatible} diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 32034f774..d76f737b5 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -135,6 +135,7 @@ bridge_v1_restart/2, bridge_v1_stop/2, bridge_v1_start/2, + bridge_v1_reset_metrics/2, %% For test cases only bridge_v1_remove/2, get_conf_root_key_if_only_one/2 @@ -1815,6 +1816,13 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> {error, not_bridge_v1_compatible} end. +bridge_v1_reset_metrics(BridgeV1Type, BridgeName) -> + BridgeV2Type = bridge_v1_type_to_bridge_v2_type(BridgeV1Type), + ConfRootKey = get_conf_root_key_if_only_one( + BridgeV2Type, BridgeName + ), + ok = reset_metrics(ConfRootKey, BridgeV2Type, BridgeName). + %%==================================================================== %% Misc helper functions %%==================================================================== From 60fab6ee45fef2f3087e26e9208b12548221f5d8 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 16 Jan 2024 12:20:35 +0100 Subject: [PATCH 50/62] refactor: attempt to improve function names --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index d76f737b5..b69882080 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1255,7 +1255,7 @@ bridge_v1_list_and_transform() -> BridgesFromActions1 ++ BridgesFromSources1. bridge_v1_lookup_and_transform(ActionType, Name) -> - case lookup_actions_or_sources(ActionType, Name) of + case lookup_in_actions_or_sources(ActionType, Name) of {ok, ConfRootKey, #{raw_config := #{<<"connector">> := ConnectorName} = RawConfig} = ActionConfig} -> BridgeV1Type = ?MODULE:bridge_v2_type_to_bridge_v1_type(ActionType, RawConfig), @@ -1287,7 +1287,7 @@ bridge_v1_lookup_and_transform(ActionType, Name) -> Error end. -lookup_actions_or_sources(ActionType, Name) -> +lookup_in_actions_or_sources(ActionType, Name) -> case lookup(?ROOT_KEY_ACTIONS, ActionType, Name) of {error, not_found} -> case lookup(?ROOT_KEY_SOURCES, ActionType, Name) of @@ -1356,7 +1356,7 @@ bridge_v1_lookup_and_transform_helper( lookup_conf(Type, Name) -> lookup_conf(?ROOT_KEY_ACTIONS, Type, Name). -lookup_conf_if_one_of_sources_actions(Type, Name) -> +lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(Type, Name) -> LookUpConfActions = lookup_conf(?ROOT_KEY_ACTIONS, Type, Name), LookUpConfSources = lookup_conf(?ROOT_KEY_SOURCES, Type, Name), case {LookUpConfActions, LookUpConfSources} of @@ -1409,7 +1409,7 @@ lookup_conf(RootName, Type, Name) -> bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> BridgeV2Type = ?MODULE:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), %% Check if the bridge v2 exists - case lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) of + case lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) of {error, _} -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, @@ -1636,7 +1636,7 @@ bridge_v1_remove(BridgeV1Type, BridgeName) -> bridge_v1_remove( ActionType, BridgeName, - lookup_conf_if_one_of_sources_actions(ActionType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(ActionType, BridgeName) ). bridge_v1_remove( @@ -1665,7 +1665,7 @@ bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) -> BridgeV2Type, BridgeName, RemoveDeps, - lookup_conf_if_one_of_sources_actions(BridgeV2Type, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, BridgeName) ). %% Bridge v1 delegated-removal in 3 steps: @@ -1760,7 +1760,7 @@ bridge_v1_enable_disable(Action, BridgeType, BridgeName) -> Action, BridgeType, BridgeName, - lookup_conf_if_one_of_sources_actions(BridgeType, BridgeName) + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeType, BridgeName) ); false -> {error, not_bridge_v1_compatible} @@ -1808,7 +1808,7 @@ bridge_v1_operation_helper(BridgeV1Type, Name, ConnectorOpFun, DoHealthCheck) -> ConfRootKey, BridgeV2Type, Name, - lookup_conf_if_one_of_sources_actions(BridgeV2Type, Name), + lookup_conf_if_exists_in_exactly_one_of_sources_and_actions(BridgeV2Type, Name), ConnectorOpFun, DoHealthCheck ); From a8f9e5676f99eedd4101ae7945fb4f721c3547a3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 10:24:19 -0300 Subject: [PATCH 51/62] docs(mqtt_bridge): add API examples --- apps/emqx_bridge/src/emqx_bridge_api.erl | 8 ++- .../src/schema/emqx_bridge_v2_schema.erl | 4 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 51 +++++++++++++++++-- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 7e929a233..f53503b86 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -49,6 +49,9 @@ -export([lookup_from_local_node/2]). -export([get_metrics_from_local_node/2]). +%% used by actions/sources schema +-export([mqtt_v1_example/1]). + %% only for testing/mocking -export([supported_versions/1]). @@ -181,7 +184,7 @@ bridge_info_examples(Method) -> }, <<"mqtt_example">> => #{ summary => <<"MQTT Bridge">>, - value => info_example(mqtt, Method) + value => mqtt_v1_example(Method) } }, emqx_enterprise_bridge_examples(Method) @@ -194,6 +197,9 @@ emqx_enterprise_bridge_examples(Method) -> emqx_enterprise_bridge_examples(_Method) -> #{}. -endif. +mqtt_v1_example(Method) -> + info_example(mqtt, Method). + info_example(Type, Method) -> maps:merge( info_example_basic(Type), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index ec9314fd2..35616ae7e 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -176,7 +176,7 @@ source_values(Method, SourceType, ConnectorType, SourceValues) -> description => <<"My example ", SourceTypeBin/binary, " source">>, connector => <>, resource_opts => #{ - health_check_interval => "30s" + health_check_interval => <<"30s">> } }, [ @@ -192,7 +192,7 @@ sources_examples(Method) -> end, Fun = fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, bridge_v2_examples, [Method]), + ConnectorExamples = erlang:apply(Module, source_examples, [Method]), lists:foldl(MergeFun, Examples, ConnectorExamples) end, SchemaModules = [Mod || {_, Mod} <- emqx_action_info:registered_schema_modules_sources()], diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index 4cf092a60..c05566234 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -24,6 +24,7 @@ -export([ bridge_v2_examples/1, + source_examples/1, conn_bridge_examples/1 ]). @@ -148,12 +149,54 @@ desc("mqtt_subscriber_source") -> desc(_) -> undefined. -bridge_v2_examples(_Method) -> +bridge_v2_examples(Method) -> [ - #{} + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_v2_schema:action_values( + Method, + _ActionType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2, + retain => false, + payload => <<"${.payload}">> + } + } + ) + } + } ]. -conn_bridge_examples(_Method) -> +source_examples(Method) -> [ - #{} + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Subscriber Source">>, + value => emqx_bridge_v2_schema:source_values( + Method, + _SourceType = mqtt, + _ConnectorType = mqtt, + #{ + parameters => #{ + topic => <<"remote/topic">>, + qos => 2 + } + } + ) + } + } + ]. + +conn_bridge_examples(Method) -> + [ + #{ + <<"mqtt">> => #{ + summary => <<"MQTT Producer Action">>, + value => emqx_bridge_api:mqtt_v1_example(Method) + } + } ]. From 2a41cad54feb20a83dcea53adacaa330fb2596bc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jan 2024 10:53:27 -0300 Subject: [PATCH 52/62] fix(sources): remote irrelevant `resource_opts` fields for sources Since they don't use buffer workers, they shouldn't have buffer-related sub-fields. --- .../src/schema/emqx_bridge_v2_schema.erl | 92 ++++++++++++++----- .../test/emqx_bridge_v2_api_SUITE.erl | 12 +-- .../emqx_bridge/test/emqx_bridge_v2_tests.erl | 4 +- apps/emqx_bridge_es/src/emqx_bridge_es.erl | 2 +- .../src/emqx_bridge_http_schema.erl | 2 +- .../src/emqx_bridge_iotdb.erl | 2 +- .../src/emqx_bridge_kafka.erl | 2 +- .../src/emqx_bridge_mongodb.erl | 2 +- .../emqx_bridge_mqtt_pubsub_action_info.erl | 2 +- .../src/emqx_bridge_mqtt_pubsub_schema.erl | 6 +- .../emqx_bridge_mqtt_v2_subscriber_SUITE.erl | 12 +-- .../src/emqx_bridge_redis_schema.erl | 2 +- 12 files changed, 84 insertions(+), 56 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 35616ae7e..5b9500156 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -53,7 +53,8 @@ -export([action_types/0, action_types_sc/0]). -export([source_types/0, source_types_sc/0]). --export([resource_opts_fields/0, resource_opts_fields/1]). +-export([action_resource_opts_fields/0, action_resource_opts_fields/1]). +-export([source_resource_opts_fields/0, source_resource_opts_fields/1]). -export([ api_fields/3 @@ -63,7 +64,8 @@ make_producer_action_schema/1, make_producer_action_schema/2, make_consumer_action_schema/1, make_consumer_action_schema/2, top_level_common_action_keys/0, - project_to_actions_resource_opts/1 + project_to_actions_resource_opts/1, + project_to_sources_resource_opts/1 ]). -export([actions_convert_from_connectors/1]). @@ -317,8 +319,10 @@ fields(actions) -> registered_schema_fields_actions(); fields(sources) -> registered_schema_fields_sources(); -fields(resource_opts) -> - resource_opts_fields(_Overrides = []). +fields(action_resource_opts) -> + action_resource_opts_fields(_Overrides = []); +fields(source_resource_opts) -> + source_resource_opts_fields(_Overrides = []). registered_schema_fields_actions() -> [ @@ -336,7 +340,9 @@ desc(actions) -> ?DESC("desc_bridges_v2"); desc(sources) -> ?DESC("desc_sources"); -desc(resource_opts) -> +desc(action_resource_opts) -> + ?DESC(emqx_resource_schema, "resource_opts"); +desc(source_resource_opts) -> ?DESC(emqx_resource_schema, "resource_opts"); desc(_) -> undefined. @@ -357,10 +363,13 @@ source_types() -> source_types_sc() -> hoconsc:enum(source_types()). -resource_opts_fields() -> - resource_opts_fields(_Overrides = []). +action_resource_opts_fields() -> + action_resource_opts_fields(_Overrides = []). -common_resource_opts_subfields() -> +source_resource_opts_fields() -> + source_resource_opts_fields(_Overrides = []). + +common_action_resource_opts_subfields() -> [ batch_size, batch_time, @@ -376,11 +385,27 @@ common_resource_opts_subfields() -> worker_pool_size ]. -common_resource_opts_subfields_bin() -> - lists:map(fun atom_to_binary/1, common_resource_opts_subfields()). +common_source_resource_opts_subfields() -> + [ + health_check_interval, + resume_interval + ]. -resource_opts_fields(Overrides) -> - ActionROFields = common_resource_opts_subfields(), +common_action_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_action_resource_opts_subfields()). + +common_source_resource_opts_subfields_bin() -> + lists:map(fun atom_to_binary/1, common_source_resource_opts_subfields()). + +action_resource_opts_fields(Overrides) -> + ActionROFields = common_action_resource_opts_subfields(), + lists:filter( + fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, + emqx_resource_schema:create_opts(Overrides) + ). + +source_resource_opts_fields(Overrides) -> + ActionROFields = common_source_resource_opts_subfields(), lists:filter( fun({Key, _Sc}) -> lists:member(Key, ActionROFields) end, emqx_resource_schema:create_opts(Overrides) @@ -404,16 +429,34 @@ make_producer_action_schema(ActionParametersRef) -> make_producer_action_schema(ActionParametersRef, _Opts = #{}). make_producer_action_schema(ActionParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, action_resource_opts)), [ {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})} - | make_consumer_action_schema(ActionParametersRef, Opts) - ]. + | common_schema(ActionParametersRef, Opts) + ] ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. -make_consumer_action_schema(ActionParametersRef) -> - make_consumer_action_schema(ActionParametersRef, _Opts = #{}). +make_consumer_action_schema(ParametersRef) -> + make_consumer_action_schema(ParametersRef, _Opts = #{}). -make_consumer_action_schema(ActionParametersRef, Opts) -> - ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, resource_opts)), +make_consumer_action_schema(ParametersRef, Opts) -> + ResourceOptsRef = maps:get(resource_opts_ref, Opts, ref(?MODULE, source_resource_opts)), + common_schema(ParametersRef, Opts) ++ + [ + {resource_opts, + mk(ResourceOptsRef, #{ + default => #{}, + desc => ?DESC(emqx_resource_schema, "resource_opts") + })} + ]. + +common_schema(ParametersRef, _Opts) -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, {connector, @@ -421,16 +464,15 @@ make_consumer_action_schema(ActionParametersRef, Opts) -> desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, {description, emqx_schema:description_schema()}, - {parameters, ActionParametersRef}, - {resource_opts, - mk(ResourceOptsRef, #{ - default => #{}, - desc => ?DESC(emqx_resource_schema, "resource_opts") - })} + {parameters, ParametersRef} ]. project_to_actions_resource_opts(OldResourceOpts) -> - Subfields = common_resource_opts_subfields_bin(), + Subfields = common_action_resource_opts_subfields_bin(), + maps:with(Subfields, OldResourceOpts). + +project_to_sources_resource_opts(OldResourceOpts) -> + Subfields = common_source_resource_opts_subfields_bin(), maps:with(Subfields, OldResourceOpts). actions_convert_from_connectors(RawConf = #{<<"actions">> := Actions}) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index fabaadb92..fc9c9573f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -455,18 +455,8 @@ source_config_base() -> <<"qos">> => 2 }, <<"resource_opts">> => #{ - <<"batch_size">> => 1, - <<"batch_time">> => <<"0ms">>, - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, <<"health_check_interval">> => <<"15s">>, - <<"inflight_window">> => 100, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"45s">>, - <<"resume_interval">> => <<"15s">>, - <<"worker_pool_size">> => <<"1">> + <<"resume_interval">> => <<"15s">> } }. diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index c64b1f2cb..a6c66c609 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -48,7 +48,9 @@ resource_opts_union_connector_actions_test() -> %% consciouly between connector and actions, in particular when/if we introduce new %% fields there. AllROFields = non_deprecated_fields(emqx_resource_schema:create_opts([])), - ActionROFields = non_deprecated_fields(emqx_bridge_v2_schema:resource_opts_fields()), + ActionROFields = non_deprecated_fields( + emqx_bridge_v2_schema:action_resource_opts_fields() + ), ConnectorROFields = non_deprecated_fields(emqx_connector_schema:resource_opts_fields()), UnionROFields = lists:usort(ConnectorROFields ++ ActionROFields), ?assertEqual( diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index b575f32ed..20a768d53 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -52,7 +52,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_create) -> [ diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index a4d956d78..b8968e82c 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -105,7 +105,7 @@ fields(action_resource_opts) -> UnsupportedOpts = [batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields("parameters_opts") -> [ diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 4be7feb19..c33ea757b 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -70,7 +70,7 @@ fields(action_resource_opts) -> fun({K, _V}) -> not lists:member(K, unsupported_opts()) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); fields(action_parameters) -> [ diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index d74ff40a1..235bc4783 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -552,7 +552,7 @@ fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(resource_opts) -> SupportedFields = [health_check_interval], - CreationOpts = emqx_bridge_v2_schema:resource_opts_fields(), + CreationOpts = emqx_bridge_v2_schema:action_resource_opts_fields(), lists:filter(fun({Field, _}) -> lists:member(Field, SupportedFields) end, CreationOpts); fields(action_field) -> {kafka_producer, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index e8eb93624..ba7be1ec4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -97,7 +97,7 @@ fields(action_parameters) -> fields(connector_resource_opts) -> emqx_connector_schema:resource_opts_fields(); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{ importance => ?IMPORTANCE_HIDDEN, converter => fun(_, _) -> 1 end, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl index cf7a5bc04..407a25118 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_action_info.erl @@ -116,7 +116,7 @@ bridge_v1_config_to_action_config_helper( ) -> %% Transform the egress part to mqtt_publisher connector config SchemaFields = emqx_bridge_mqtt_pubsub_schema:fields("mqtt_subscriber_source"), - ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(action_resource_opts), + ResourceOptsSchemaFields = emqx_bridge_mqtt_pubsub_schema:fields(source_resource_opts), ConfigMap1 = general_action_conf_map_from_bridge_v1_config( Config, ConnectorName, SchemaFields, ResourceOptsSchemaFields ), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl index c05566234..05b2d6d3a 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_pubsub_schema.erl @@ -105,8 +105,10 @@ fields(action_resource_opts) -> UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, - emqx_bridge_v2_schema:resource_opts_fields() + emqx_bridge_v2_schema:action_resource_opts_fields() ); +fields(source_resource_opts) -> + emqx_bridge_v2_schema:source_resource_opts_fields(); fields(Field) when Field == "get_bridge_v2"; Field == "post_bridge_v2"; @@ -132,6 +134,8 @@ desc("config") -> ?DESC("desc_config"); desc(action_resource_opts) -> ?DESC(emqx_resource_schema, "creation_opts"); +desc(source_resource_opts) -> + ?DESC(emqx_resource_schema, "creation_opts"); desc(action_parameters) -> ?DESC(action_parameters); desc(ingress_parameters) -> diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl index a0b3edfa7..3e5471d55 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_v2_subscriber_SUITE.erl @@ -112,18 +112,8 @@ source_config(Overrides0) -> <<"qos">> => 2 }, <<"resource_opts">> => #{ - <<"batch_size">> => 1, - <<"batch_time">> => <<"0ms">>, - <<"buffer_mode">> => <<"memory_only">>, - <<"buffer_seg_bytes">> => <<"10MB">>, <<"health_check_interval">> => <<"15s">>, - <<"inflight_window">> => 100, - <<"max_buffer_bytes">> => <<"256MB">>, - <<"metrics_flush_interval">> => <<"1s">>, - <<"query_mode">> => <<"sync">>, - <<"request_ttl">> => <<"45s">>, - <<"resume_interval">> => <<"15s">>, - <<"worker_pool_size">> => <<"1">> + <<"resume_interval">> => <<"15s">> } }, maps:merge(CommonConfig, Overrides). diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl index 9373fe8bd..adda91f37 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_schema.erl @@ -76,7 +76,7 @@ fields(redis_action) -> [ResOpts] = emqx_connector_schema:resource_opts_ref(?MODULE, action_resource_opts), lists:keyreplace(resource_opts, 1, Schema, ResOpts); fields(action_resource_opts) -> - emqx_bridge_v2_schema:resource_opts_fields([ + emqx_bridge_v2_schema:action_resource_opts_fields([ {batch_size, #{desc => ?DESC(batch_size)}}, {batch_time, #{desc => ?DESC(batch_time)}} ]); From 01d52e37c44303dfb01b888fb960f7473c5c9ea6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 16 Jan 2024 17:35:22 +0100 Subject: [PATCH 53/62] fix: resource tag type should be binary string Co-authored-by: Thales Macedo Garitezi --- apps/emqx/src/emqx_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 66520df75..33d027c19 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3829,7 +3829,7 @@ description_schema() -> tags_schema() -> sc( - hoconsc:array(string()), + hoconsc:array(binary()), #{ desc => ?DESC(resource_tags), required => false, From 238ecc68cf4a89a2a6c32f9b0786dcf815147f10 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 16 Jan 2024 22:43:01 +0800 Subject: [PATCH 54/62] fix(iotdb): enhances type checking when converting value --- .../src/emqx_bridge_iotdb_connector.erl | 46 ++++++++++--------- .../test/emqx_bridge_iotdb_impl_SUITE.erl | 11 ++++- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index fec50f779..d7b45c18d 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -444,19 +444,22 @@ proc_data( DataType = list_to_binary( string:uppercase(binary_to_list(emqx_placeholder:proc_tmpl(DataType0, Msg))) ), - case proc_value(DataType, ValueTkn, Msg) of - {ok, Value} -> - proc_data(T, Msg, Nows, [ - #{ - timestamp => iot_timestamp(TimestampTkn, Msg, Nows), - measurement => emqx_placeholder:proc_tmpl(Measurement, Msg), - data_type => DataType, - value => Value - } - | Acc - ]); - Error -> - Error + try + proc_data(T, Msg, Nows, [ + #{ + timestamp => iot_timestamp(TimestampTkn, Msg, Nows), + measurement => emqx_placeholder:proc_tmpl(Measurement, Msg), + data_type => DataType, + value => proc_value(DataType, ValueTkn, Msg) + } + | Acc + ]) + catch + throw:Reason -> + {error, Reason}; + Error:Reason:Stacktrace -> + ?SLOG(debug, #{exception => Error, reason => Reason, stacktrace => Stacktrace}), + {error, invalid_data} end; proc_data([], _Msg, _Nows, Acc) -> {ok, lists:reverse(Acc)}. @@ -478,19 +481,18 @@ iot_timestamp(Timestamp, _) when is_binary(Timestamp) -> binary_to_integer(Timestamp). proc_value(<<"TEXT">>, ValueTkn, Msg) -> - {ok, - case emqx_placeholder:proc_tmpl(ValueTkn, Msg) of - <<"undefined">> -> null; - Val -> Val - end}; + case emqx_placeholder:proc_tmpl(ValueTkn, Msg) of + <<"undefined">> -> null; + Val -> Val + end; proc_value(<<"BOOLEAN">>, ValueTkn, Msg) -> - {ok, convert_bool(replace_var(ValueTkn, Msg))}; + convert_bool(replace_var(ValueTkn, Msg)); proc_value(Int, ValueTkn, Msg) when Int =:= <<"INT32">>; Int =:= <<"INT64">> -> - {ok, convert_int(replace_var(ValueTkn, Msg))}; + convert_int(replace_var(ValueTkn, Msg)); proc_value(Int, ValueTkn, Msg) when Int =:= <<"FLOAT">>; Int =:= <<"DOUBLE">> -> - {ok, convert_float(replace_var(ValueTkn, Msg))}; + convert_float(replace_var(ValueTkn, Msg)); proc_value(Type, _, _) -> - {error, {invalid_type, Type}}. + throw(#{reason => invalid_type, type => Type}). replace_var(Tokens, Data) when is_list(Tokens) -> [Val] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}), diff --git a/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl index 8145faf33..1093993b2 100644 --- a/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl +++ b/apps/emqx_bridge_iotdb/test/emqx_bridge_iotdb_impl_SUITE.erl @@ -664,7 +664,16 @@ t_sync_query_invalid_type(Config) -> DeviceId = iotdb_device(Config), Payload = make_iotdb_payload(DeviceId, "temp", "IxT32", "36"), MakeMessageFun = make_message_fun(iotdb_topic(Config), Payload), - IsInvalidType = fun(Result) -> ?assertMatch({error, {invalid_type, _}}, Result) end, + IsInvalidType = fun(Result) -> ?assertMatch({error, #{reason := invalid_type}}, Result) end, + ok = emqx_bridge_v2_testlib:t_sync_query( + Config, MakeMessageFun, IsInvalidType, iotdb_bridge_on_query + ). + +t_sync_query_unmatched_type(Config) -> + DeviceId = iotdb_device(Config), + Payload = make_iotdb_payload(DeviceId, "temp", "BOOLEAN", "not boolean"), + MakeMessageFun = make_message_fun(iotdb_topic(Config), Payload), + IsInvalidType = fun(Result) -> ?assertMatch({error, invalid_data}, Result) end, ok = emqx_bridge_v2_testlib:t_sync_query( Config, MakeMessageFun, IsInvalidType, iotdb_bridge_on_query ). From b2af7fdd704dc005e6e14b39d2e0c29d6a65479e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 17 Jan 2024 12:51:10 +0800 Subject: [PATCH 55/62] fix(sysk): fix a update issue for the Syskeeper forwarder --- .../src/emqx_bridge_syskeeper_connector.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index a5c59e82a..ff05a58d8 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -123,8 +123,11 @@ fields(Field) when Field == "post"; Field == "put" -> + Fields = + fields("connection_fields") ++ + emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts), emqx_connector_schema:api_fields( - Field ++ "_connector", ?CONNECTOR_TYPE, fields("connection_fields") + Field ++ "_connector", ?CONNECTOR_TYPE, Fields ). desc(config) -> From 85b6a3454c8c71d0421b82f6c04e282afd8f4aff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 18 Jan 2024 08:36:55 +0100 Subject: [PATCH 56/62] fix(authz): use binary() type instead of string() for cache.excludes --- apps/emqx/src/emqx_authz_cache.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_authz_cache.erl b/apps/emqx/src/emqx_authz_cache.erl index 016c720ed..8dcc1827a 100644 --- a/apps/emqx/src/emqx_authz_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -56,7 +56,7 @@ drain_k() -> {?MODULE, drain_timestamp}. -spec is_enabled(emqx_types:topic()) -> boolean(). is_enabled(Topic) -> case emqx:get_config([authorization, cache]) of - #{enable := true, excludes := Filters} -> + #{enable := true, excludes := Filters} when Filters =/= [] -> not is_excluded(Topic, Filters); #{enable := IsEnabled} -> IsEnabled diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 33d027c19..ae22db14f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -468,7 +468,7 @@ fields(authz_cache) -> } )}, {excludes, - sc(hoconsc:array(string()), #{ + sc(hoconsc:array(binary()), #{ default => [], desc => ?DESC(fields_authz_cache_excludes) })} From 556092b7d0935ebe0e20a70b9e3092615c1bf459 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 18 Jan 2024 08:33:57 +0100 Subject: [PATCH 57/62] feat(authz/prometheus): add authz cache_miss counter --- apps/emqx/src/emqx_access_control.erl | 5 ++++- apps/emqx/src/emqx_metrics.erl | 4 +++- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 6e8f9b181..b786e2c18 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -135,6 +135,7 @@ is_username_defined(_) -> false. check_authorization_cache(ClientInfo, Action, Topic) -> case emqx_authz_cache:get_authz_cache(Action, Topic) of not_found -> + inc_authz_metrics(cache_miss), AuthzResult = do_authorize(ClientInfo, Action, Topic), emqx_authz_cache:put_authz_cache(Action, Topic, AuthzResult), AuthzResult; @@ -219,7 +220,9 @@ inc_authz_metrics(allow) -> inc_authz_metrics(deny) -> emqx_metrics:inc('authorization.deny'); inc_authz_metrics(cache_hit) -> - emqx_metrics:inc('authorization.cache_hit'). + emqx_metrics:inc('authorization.cache_hit'); +inc_authz_metrics(cache_miss) -> + emqx_metrics:inc('authorization.cache_miss'). inc_authn_metrics(error) -> emqx_metrics:inc('authentication.failure'); diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 40427c192..e00983bfa 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -258,7 +258,8 @@ -define(STASTS_ACL_METRICS, [ {counter, 'authorization.allow'}, {counter, 'authorization.deny'}, - {counter, 'authorization.cache_hit'} + {counter, 'authorization.cache_hit'}, + {counter, 'authorization.cache_miss'} ]). %% Statistic metrics for auth checking @@ -702,6 +703,7 @@ reserved_idx('session.terminated') -> 224; reserved_idx('authorization.allow') -> 300; reserved_idx('authorization.deny') -> 301; reserved_idx('authorization.cache_hit') -> 302; +reserved_idx('authorization.cache_miss') -> 303; reserved_idx('authentication.success') -> 310; reserved_idx('authentication.success.anonymous') -> 311; reserved_idx('authentication.failure') -> 312; diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 599e20fb7..fe0c42566 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.18"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 09ba157a0..d513e2c37 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -486,6 +486,8 @@ emqx_collect(emqx_authorization_deny, Stats) -> counter_metric(?C('authorization.deny', Stats)); emqx_collect(emqx_authorization_cache_hit, Stats) -> counter_metric(?C('authorization.cache_hit', Stats)); +emqx_collect(emqx_authorization_cache_miss, Stats) -> + counter_metric(?C('authorization.cache_miss', Stats)); emqx_collect(emqx_authorization_superuser, Stats) -> counter_metric(?C('authorization.superuser', Stats)); emqx_collect(emqx_authorization_nomatch, Stats) -> @@ -591,6 +593,7 @@ emqx_metrics_acl() -> emqx_authorization_allow, emqx_authorization_deny, emqx_authorization_cache_hit, + emqx_authorization_cache_miss, emqx_authorization_superuser, emqx_authorization_nomatch, emqx_authorization_matched_allow, From 7802d6e0186c236384d627418f503dec1c476855 Mon Sep 17 00:00:00 2001 From: aiotter Date: Wed, 17 Jan 2024 03:45:16 +0900 Subject: [PATCH 58/62] chore: fix typos --- apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl | 2 +- apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src | 2 +- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl index 63da372ff..e543e3c03 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt_schema.erl @@ -137,7 +137,7 @@ secret_base64_encoded(_) -> undefined. public_key(type) -> string(); public_key(desc) -> ?DESC(?FUNCTION_NAME); -public_key(required) -> ture; +public_key(required) -> true; public_key(_) -> undefined. endpoint(type) -> string(); diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index 19f3bf552..7f5e8f04a 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl index 42a0e8f74..f47aebada 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl @@ -163,7 +163,7 @@ convert_certs( ) -> case emqx_tls_lib:ensure_ssl_files( - Dir, #{enable => ture, certfile => Cert, keyfile => Key}, #{} + Dir, #{enable => true, certfile => Cert, keyfile => Key}, #{} ) of {ok, #{certfile := CertPath, keyfile := KeyPath}} -> From 7f5fe9190565157cb381c75a72e0e4127fa94b8c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jan 2024 17:14:28 +0800 Subject: [PATCH 59/62] fix: es's action is atom not binary --- .../docker-compose-elastic-search-tls.yaml | 101 ++++++++++++++++++ apps/emqx_bridge_es/src/emqx_bridge_es.erl | 6 +- .../src/emqx_bridge_es_connector.erl | 5 +- .../src/emqx_bridge_iotdb_connector.erl | 4 +- 4 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-elastic-search-tls.yaml diff --git a/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml b/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml new file mode 100644 index 000000000..fef93d785 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml @@ -0,0 +1,101 @@ +version: "3.9" + +services: + setup: + image: public.ecr.aws/elastic/elasticsearch:${ELASTIC_TAG} + volumes: + - ./elastic:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/ca/ca.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + es01: + depends_on: + setup: + condition: service_healthy + image: public.ecr.aws/elastic/elasticsearch:${ELASTIC_TAG} + volumes: + - ./elastic:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + environment: + - node.name=es01 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.license.self_generated.type=${LICENSE} + mem_limit: 1073741824 + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + es01: + condition: service_healthy + image: public.ecr.aws/elastic/kibana:${ELASTIC_TAG} + volumes: + - ./elastic:/usr/share/kibana/config/certs + - kibanadata:/usr/share/kibana/data + ports: + - 5601:5601 + environment: + - SERVERNAME=kibana + - ELASTICSEARCH_HOSTS=https://es01:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + mem_limit: 1073741824 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + +volumes: + esdata01: + driver: local + kibanadata: + driver: local diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index b575f32ed..9975eb1b1 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -98,10 +98,14 @@ action_union_member_selector({value, Value}) -> [?R_REF(action_delete)]; #{<<"action">> := <<"update">>} -> [?R_REF(action_update)]; - _ -> + #{<<"action">> := Action} when is_atom(Action) -> + Value1 = Value#{<<"action">> => atom_to_binary(Action)}, + action_union_member_selector({value, Value1}); + Actual -> Expected = "create | delete | update", throw(#{ field_name => action, + actual => Actual, expected => Expected }) end. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index fe86eac56..7e49aeb55 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -200,7 +200,7 @@ on_start(InstanceId, Config) -> ?SLOG(info, #{ msg => "elasticsearch_bridge_started", instance_id => InstanceId, - request => maps:get(request, State, <<>>) + request => emqx_utils:redact(maps:get(request, State, <<>>)) }), ?tp(elasticsearch_bridge_started, #{instance_id => InstanceId}), {ok, State#{channels => #{}}}; @@ -208,7 +208,7 @@ on_start(InstanceId, Config) -> ?SLOG(error, #{ msg => "failed_to_start_elasticsearch_bridge", instance_id => InstanceId, - request => maps:get(request, Config, <<>>), + request => emqx_utils:redact(maps:get(request, Config, <<>>)), reason => Reason }), throw(failed_to_start_elasticsearch_bridge) @@ -354,6 +354,7 @@ add_query_string(Keys, Param0) -> end. to_str(List) when is_list(List) -> List; +to_str(Bin) when is_binary(Bin) -> binary_to_list(Bin); to_str(false) -> "false"; to_str(true) -> "true"; to_str(Atom) when is_atom(Atom) -> atom_to_list(Atom). diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index d7b45c18d..ccf97f143 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -212,7 +212,7 @@ on_start(InstanceId, #{iotdb_version := Version} = Config) -> ?SLOG(info, #{ msg => "iotdb_bridge_started", instance_id => InstanceId, - request => maps:get(request, State, <<>>) + request => emqx_utils:redact(maps:get(request, State, <<>>)) }), ?tp(iotdb_bridge_started, #{instance_id => InstanceId}), {ok, State#{iotdb_version => Version, channels => #{}}}; @@ -220,7 +220,7 @@ on_start(InstanceId, #{iotdb_version := Version} = Config) -> ?SLOG(error, #{ msg => "failed_to_start_iotdb_bridge", instance_id => InstanceId, - request => maps:get(request, Config, <<>>), + request => emqx_utils:redact(maps:get(request, Config, <<>>)), reason => Reason }), throw(failed_to_start_iotdb_bridge) From 91368a57ff12500309284a545807e124ac30e534 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 18 Jan 2024 08:56:39 +0800 Subject: [PATCH 60/62] test: add es docker CI test --- .ci/docker-compose-file/.env | 9 + .../docker-compose-elastic-search-tls.yaml | 8 + .ci/docker-compose-file/elastic/ca/ca.crt | 20 + .ci/docker-compose-file/elastic/ca/ca.key | 27 ++ .ci/docker-compose-file/elastic/es01/es01.crt | 20 + .ci/docker-compose-file/elastic/es01/es01.key | 27 ++ .ci/docker-compose-file/elastic/instances.yml | 7 + .ci/docker-compose-file/toxiproxy.json | 6 + apps/emqx_bridge_es/docker-ct | 1 + apps/emqx_bridge_es/src/emqx_bridge_es.erl | 8 +- .../test/emqx_bridge_es_SUITE.erl | 372 ++++++++++++++++++ .../test/emqx_bridge_es_SUITE_data/es.crt | 20 + apps/emqx_connector/src/emqx_connector.erl | 3 + rel/i18n/emqx_bridge_es.hocon | 2 +- scripts/ct/run.sh | 3 + 15 files changed, 528 insertions(+), 5 deletions(-) create mode 100644 .ci/docker-compose-file/elastic/ca/ca.crt create mode 100644 .ci/docker-compose-file/elastic/ca/ca.key create mode 100644 .ci/docker-compose-file/elastic/es01/es01.crt create mode 100644 .ci/docker-compose-file/elastic/es01/es01.key create mode 100644 .ci/docker-compose-file/elastic/instances.yml create mode 100644 apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl create mode 100644 apps/emqx_bridge_es/test/emqx_bridge_es_SUITE_data/es.crt diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 3be2b7415..73ec47d00 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -16,4 +16,13 @@ HSTREAMDB_ZK_TAG=3.8.1 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server SQLSERVER_TAG=2019-CU19-ubuntu-20.04 + +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD="emqx123" +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD="emqx123" +# Version of Elastic products +ELASTIC_TAG=8.11.4 +LICENSE=basic + TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml b/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml index fef93d785..50491a88a 100644 --- a/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml +++ b/.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml @@ -36,6 +36,8 @@ services: setup: condition: service_healthy image: public.ecr.aws/elastic/elasticsearch:${ELASTIC_TAG} + container_name: elasticsearch + hostname: elasticsearch volumes: - ./elastic:/usr/share/elasticsearch/config/certs - esdata01:/usr/share/elasticsearch/data @@ -66,6 +68,9 @@ services: interval: 10s timeout: 10s retries: 120 + restart: always + networks: + - emqx_bridge kibana: depends_on: @@ -93,6 +98,9 @@ services: interval: 10s timeout: 10s retries: 120 + restart: always + networks: + - emqx_bridge volumes: esdata01: diff --git a/.ci/docker-compose-file/elastic/ca/ca.crt b/.ci/docker-compose-file/elastic/ca/ca.crt new file mode 100644 index 000000000..2d47d555c --- /dev/null +++ b/.ci/docker-compose-file/elastic/ca/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAIrN275DCtGnotTPpxwvQ5751N4OMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTI0MDExNjAyMzIyMFoXDTI3MDExNTAyMzIyMFowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCy0nwiEurUkIPFMLV1weVM +pPk/AlwZUzqjkeL44gsY53XI9Q05w/sL9u6PzwrXgTCFWNXzI9+MoAtp8phPkn14 +cmg5/3sLe9YcFVFjYK/MoljlUbPDj+4dgk8l+w5FRSi0+JN5krUm7rYk9lojAkeS +fX8RU7ekKGbjBXIFtPxX5GNadu9RidR5GkHM3XroAIoris8bFOzMgFn9iybYnkhq +0S+Hpv0A8FVxzle0KNbPpsIkxXH2DnP2iPTDym9xJNl9Iv9MPtj9XaamH7TmXcSt +MbjkAudKsCw4bRuhHonM16DIUr8sX5UcRcAWyJ1x1qpZaOzMdh2VdYAHNuOsZwzJ +AgMBAAGjUzBRMB0GA1UdDgQWBBTAyDlp8NZfPe8NCGVlHJSVclGOhTAfBgNVHSME +GDAWgBTAyDlp8NZfPe8NCGVlHJSVclGOhTAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQAeIUXRKmC53iirY4P49YspLafspAMf4ndMFQAp+Oc223Vs +hQC4axNoYnUdzWDH6LioAN7P826xNPqtXvTZF9fmeX7K8Nm9Kdj+for+QQI3j6+X +zq98VVkACb8b/Mc9Nac/WBbv/1IKyKgNNta7//WNPgAFolOfti/C0NLsPcKhrM9L +mGbvRX8ZjH8pVJ0YTy4/xfDcF7G/Lxl4Yvb0ZXpuQbvE1+Y0h5aoTNshT/skJxC4 +iyVseYr21s3pptKcr6H9KZuSdZe5pbEo+81nT15w+50aswFLk9GCYh5UsQ+1jkRK +cKgxP93i6x8BVbQJGKi1A1jhauSKX2IpWZQsHy4p +-----END CERTIFICATE----- diff --git a/.ci/docker-compose-file/elastic/ca/ca.key b/.ci/docker-compose-file/elastic/ca/ca.key new file mode 100644 index 000000000..72786207a --- /dev/null +++ b/.ci/docker-compose-file/elastic/ca/ca.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAstJ8IhLq1JCDxTC1dcHlTKT5PwJcGVM6o5Hi+OILGOd1yPUN +OcP7C/buj88K14EwhVjV8yPfjKALafKYT5J9eHJoOf97C3vWHBVRY2CvzKJY5VGz +w4/uHYJPJfsORUUotPiTeZK1Ju62JPZaIwJHkn1/EVO3pChm4wVyBbT8V+RjWnbv +UYnUeRpBzN166ACKK4rPGxTszIBZ/Ysm2J5IatEvh6b9APBVcc5XtCjWz6bCJMVx +9g5z9oj0w8pvcSTZfSL/TD7Y/V2mph+05l3ErTG45ALnSrAsOG0boR6JzNegyFK/ +LF+VHEXAFsidcdaqWWjszHYdlXWABzbjrGcMyQIDAQABAoIBAAZOLXYanmjpIRpX +h7h7oikYEplWDRcQBBvvKZaOyuchhznTKTiZmF0xQ3Ny8J4Ndj9ndODWSZxI6uod +FaGNp+qytwnfgDBVGSVDm6tyRfSkX1fTsA/j3/iupvmO/w9yezdZYgLaCVTyex31 +yVMdchZgYjYDUpEBYzJbV2xL18+GBRmmPjdXumlpcJqcclxjOQJSu/1WCGVfn/e/ +64NQpAm7NSKLqeUl32g0/DvUpmYRfmf7ZjVUjePaJQU6sw5/N+3V9F1hYs8VSWz0 +OMzYIfUcvixw+VWx5bu0nWt98FirhsQPjCTThD+DHP6koXGrdXpeMOQE1YZmoV5T +vP0X+FECgYEA5dsKVDQFL67muqz3CNRVM0xDWACCoa8789hYoxvhd1iO3e4kwXBa +ABPcZckioq+HiQ4UIxC2AhQ1FuTeIUTq7LZ0HtAAdKFi48U4LzmPhNUpG1E/HbJ3 +GQbi4u1cAzGYuhdywktgBhn9bJ4XB7+X3815Y9qKkuRcwtXgKGDy8HkCgYEAxyly +vc7NBkLfIAmkOsm6VXfvfBTEUBUGi6+k1rarTUxWFIgRuk4FHywwWUTdxWBKJz3n +HNNJb/g7CcufdhLTuWVHQtJDxYf2cJjoi+Kf7/i/Qs9Nyhokj5Mnh6KlZQOWXpZd +Gwn/O13NeDxt1TIVO2xp6zY4FhVEPvaHuxsMCtECgYA7/eR/P6iO3nZoCJbdXhXy +spftEw0FSCg8p53SzIcXUCzRrcM4HavP0181zb5VebzFP8Bvun/WoRGOLSPwyP0L +1T8Pf7huuGSIEERuxvY3dC8raxQvGxJMnOiA0/Ss/Lfg8hfIsEWashPb0pMuOYpZ +JlblgfejCSlQzOOZhlxB+QKBgQCKmizRLV9/0QAJAsy5YPR9UJdpCebJOKiyg806 +5Ct5AvwRE9UKjAuCczU+mu+f0fApOSpi5CQCeYVUvtG90UJpjrM2LLCfgoyeNbv4 +xgG6dqlcbHrdgK4bATUMbsOd9g4qy4gGLkHi5df9qkhhi5Y9Iajg2X3U2H4DN3yk +WSFbUQKBgQCLz333qWOuT3OBv+EYxHDQUS4YG+dReUos+v0iPJzu+spnfibBF5IC +RjHIhPsdN1byNB0naXOkkz4tUlLGXv6umFgDtQvy/2rxvxQmUGp/WY1VM2+164Xe +NEWdMEU6UckCoMO77kw8JosKhmXCYaSW5bWwnXuEpOj9WWpwjKtxlA== +-----END RSA PRIVATE KEY----- diff --git a/.ci/docker-compose-file/elastic/es01/es01.crt b/.ci/docker-compose-file/elastic/es01/es01.crt new file mode 100644 index 000000000..002f5727b --- /dev/null +++ b/.ci/docker-compose-file/elastic/es01/es01.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAiigAwIBAgIUe90yOBN1KBxOEr2jro3epamZksIwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMjQwMTE2MDIzMjIyWhcNMjcwMTE1MDIzMjIyWjAPMQ0wCwYD +VQQDEwRlczAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxGEL71pV +j8qoUxEuL7qjRSeS1eHxeKhu2jqEZb7iA1o/7b/26QuYAkoYL+WuJNfYjg5F/O8W +VVuAYIlN6a/mC6wT2t3pX4YSrdp+i3gtAC/LX+8mAeqMQPD+4jitOwjOsYzbuFCb +nYl86dnFPl/+Pmj20mtZ+Wt7oIPD88j6+r5qgv59pHICxS7Cq304LDTRQbNoT8HO +4c9VGGGtWIdtrqiYrz1OVefkffMrvFt77v6dKHn8g5tSyfQUDCoEKtTOc3Pe5zCB +vIMs6HaapoSkl8XdpFHQ712PCZRebAMCrVcPYQ3r8e9GYmLY/NhxEn3dWTqRhHeg +UD13O8o1aBWonwIDAQABo28wbTAdBgNVHQ4EFgQUXvGJtSf2/mLOK17AzUridtCV +xWwwHwYDVR0jBBgwFoAUwMg5afDWXz3vDQhlZRyUlXJRjoUwIAYDVR0RBBkwF4IJ +bG9jYWxob3N0hwR/AAABggRlczAxMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQAD +ggEBACaNq3ZqrbsGvbEtrf6kJGIsTokTFHeVJUSYmt1ZZzDFLSepXAC/J8gphV45 +B+YSlkDPNTwMYlf7TUYY872zkdqOXN9r0NUx8MzVAX0+rux0RJba5GGUvJGZDNMX +WM5z9ry1KjQSQ1bSoRQOD3QArmBmhvikHjLc97Vqt56N0wA/ztXWOpNZX/TXmast +aXlUbcfQE73Cdq9tW1ATXwbQ2Gf7vVAUT3zjZSZbNdgPuBicGJHf85Fhjm2ND4+R +sjLIOQ2YgVxNHYbueScc6lJM5RNK194K7WrEQnRyGHT3NaDUm0FFNl//aQeq1ZVw +6gaUYlkTFauXwEYMDK901cWFaBE= +-----END CERTIFICATE----- diff --git a/.ci/docker-compose-file/elastic/es01/es01.key b/.ci/docker-compose-file/elastic/es01/es01.key new file mode 100644 index 000000000..b401c5376 --- /dev/null +++ b/.ci/docker-compose-file/elastic/es01/es01.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAxGEL71pVj8qoUxEuL7qjRSeS1eHxeKhu2jqEZb7iA1o/7b/2 +6QuYAkoYL+WuJNfYjg5F/O8WVVuAYIlN6a/mC6wT2t3pX4YSrdp+i3gtAC/LX+8m +AeqMQPD+4jitOwjOsYzbuFCbnYl86dnFPl/+Pmj20mtZ+Wt7oIPD88j6+r5qgv59 +pHICxS7Cq304LDTRQbNoT8HO4c9VGGGtWIdtrqiYrz1OVefkffMrvFt77v6dKHn8 +g5tSyfQUDCoEKtTOc3Pe5zCBvIMs6HaapoSkl8XdpFHQ712PCZRebAMCrVcPYQ3r +8e9GYmLY/NhxEn3dWTqRhHegUD13O8o1aBWonwIDAQABAoIBADJ3A/Om4az5dcce +96EBU9q+IDBBh2Wr1wzSk9p3sqoM47fLqH5b4dzYwJ1yZw2FwFtFFLw6jqExyexE +7JY8gyAFwPZyJ3pKQHuX1gQuRlYxchB9quU8Kn230LA+w1mT2lXrLj2PzWWvAsAv +m837KiFMpP0O5EjB07u8kLsRr1mG6QQ24Kc8oxd7xLXIiPzSvsOpYwo9hmIWENd5 +kyA7oSa9EmN3TRTkKOHI7cFQ3DqIGdO71waUofKOdx39DyHS2YKWxDE/LUjkS9zw +1AyZG09l4uowyLRqwYhivEq9Za6rdc64yheuHatAM9kC2AOcVcsCPZquIe90k4t1 +L7e9CAECgYEA1W483xTW8ngzxv9MMuPiW+PwVGRpyQrbO6OZOxdWEYfhrZlk5wlW +XK2T85jqooJwMWPTk1F49vZ9WN2KuLkL65GlkEtkFbxmOiFJjXuWwycbFSk05hPs +4AESBYHieaSPcwYhvLeG6g4PFyeqmbAGnKsJaj2ylPwDBOc7LgVlqAECgYEA64wo +gZwaj5SlP8M/OqGH04UVYr1kP/Eq6eiDfMyV5exy+pyzofZyNKUfJfw6sGgyRRHx +OVxlnPMsZ8zbdOXsvUEIeavpwDfQcp5eAURL65I6GMLsx2QpfiN2mDe1MqQW0jct +UleFaURgS84KHLE0+tBBg906jOHGjsE7Q3lyUJ8CgYBYYPev4K9JZGD8bEcfY6Ie +Lvsb1yC+8VHrFkmjYHxxcfUPr89KpGEwq2fynUW72YufyBiajkgq69Ln84U4DNhU +ydDnOXDOV191fsc4YQ8C7LSYRKH1DBcwgwD1at1fRbdpCAb8YHrrfLre+bv5PBzg +zyps5fOHIfwWEbI90lpQAQKBgQDoMMqBMTtxi+r1lucOScrVtFuncOCQs5BE8cIj +1JxzAQk6iBv/LSvZP2gcDq5f1Oaw9YXfsHguJfwA+ozeiAQ9bw0Gu3N52sstIXWz +M/rO5d9FJ2k3CEJqqFSwqkGBAQXKBUA06jeF1DREpX+MVxbNo1rhvMOJusn7UPm1 +gtMwKwKBgQCfRzFO10ITwrw8rcRZwO9Axgqf11V7xn6qpgRxj4h0HOErVTCN1H0b +vE3Pz7cxS/g9vFRP37TuqBLfGVzPt9LAEFwCWPeZJLROBLHyu8XrhTbQx+sI2/pe +SBEJAQAHtYasFTE0sBEKNEY2rIt1c29XZhyhhtNKD9gRN/gB355wLg== +-----END RSA PRIVATE KEY----- diff --git a/.ci/docker-compose-file/elastic/instances.yml b/.ci/docker-compose-file/elastic/instances.yml new file mode 100644 index 000000000..bf1ad4102 --- /dev/null +++ b/.ci/docker-compose-file/elastic/instances.yml @@ -0,0 +1,7 @@ +instances: + - name: es01 + dns: + - es01 + - localhost + ip: + - 127.0.0.1 diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 4b2b6ccf2..c58474039 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -191,5 +191,11 @@ "listen": "0.0.0.0:636", "upstream": "ldap:636", "enabled": true + }, + { + "name": "elasticsearch", + "listen": "0.0.0.0:9200", + "upstream": "elasticsearch:9200", + "enabled": true } ] diff --git a/apps/emqx_bridge_es/docker-ct b/apps/emqx_bridge_es/docker-ct index 80f0d394b..f7cd9ab28 100644 --- a/apps/emqx_bridge_es/docker-ct +++ b/apps/emqx_bridge_es/docker-ct @@ -1 +1,2 @@ toxiproxy +elasticsearch diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index 9975eb1b1..569b67dba 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -59,7 +59,7 @@ fields(action_create) -> action(create), index(), id(false), - doc(true), + doc(), routing(), require_alias(), overwrite() @@ -72,7 +72,7 @@ fields(action_update) -> action(update), index(), id(true), - doc(true), + doc(), routing(), require_alias() | http_common_opts() @@ -153,12 +153,12 @@ id(Required) -> } )}. -doc(Required) -> +doc() -> {doc, ?HOCON( binary(), #{ - required => Required, + required => false, example => <<"${payload.doc}">>, desc => ?DESC("config_parameters_doc") } diff --git a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl new file mode 100644 index 000000000..e7e2dba28 --- /dev/null +++ b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl @@ -0,0 +1,372 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_es_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(TYPE, elasticsearch). +-define(CA, "es.crt"). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + ProxyName = "elasticsearch", + ESHost = os:getenv("ELASTICSEARCH_HOST", "elasticsearch"), + ESPort = list_to_integer(os:getenv("ELASTICSEARCH_PORT", "9200")), + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_es, + emqx_bridge, + emqx_rule_engine, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + {ok, _} = emqx_common_test_http:create_default_app(), + wait_until_elasticsearch_is_up(ESHost, ESPort), + [ + {apps, Apps}, + {proxy_name, ProxyName}, + {es_host, ESHost}, + {es_port, ESPort} + | Config + ]. + +es_checks() -> + case os:getenv("IS_CI") of + "yes" -> 10; + _ -> 1 + end. + +wait_until_elasticsearch_is_up(Host, Port) -> + wait_until_elasticsearch_is_up(es_checks(), Host, Port). + +wait_until_elasticsearch_is_up(0, Host, Port) -> + throw({{Host, Port}, not_available}); +wait_until_elasticsearch_is_up(Count, Host, Port) -> + timer:sleep(1000), + case emqx_common_test_helpers:is_all_tcp_servers_available([{Host, Port}]) of + true -> ok; + false -> wait_until_elasticsearch_is_up(Count - 1, Host, Port) + end. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + %ProxyHost = ?config(proxy_host, Config), + %ProxyPort = ?config(proxy_port, Config), + %emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + emqx_cth_suite:stop(Apps), + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + %ProxyHost = ?config(proxy_host, Config), + %ProxyPort = ?config(proxy_port, Config), + %emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), + emqx_common_test_helpers:call_janitor(60_000), + ok. + +%%------------------------------------------------------------------------------------- +%% Helper fns +%%------------------------------------------------------------------------------------- + +check_send_message_with_action(ActionName, ConnectorName) -> + #{payload := _Payload} = send_message(ActionName), + %% ###################################### + %% Check if message is sent to es + %% ###################################### + check_action_metrics(ActionName, ConnectorName). + +send_message(ActionName) -> + %% ###################################### + %% Create message + %% ###################################### + Time = erlang:unique_integer(), + BinTime = integer_to_binary(Time), + Payload = #{<<"name">> => <<"emqx">>, <<"release_time">> => BinTime}, + Index = <<"emqx-test-index">>, + Msg = #{ + clientid => BinTime, + payload => Payload, + timestamp => Time, + index => Index + }, + %% ###################################### + %% Send message + %% ###################################### + emqx_bridge_v2:send_message(?TYPE, ActionName, Msg, #{}), + #{payload => Payload}. + +check_action_metrics(ActionName, ConnectorName) -> + ActionId = emqx_bridge_v2:id(?TYPE, ActionName, ConnectorName), + Metrics = + #{ + match => emqx_resource_metrics:matched_get(ActionId), + failed => emqx_resource_metrics:failed_get(ActionId), + queuing => emqx_resource_metrics:queuing_get(ActionId), + dropped => emqx_resource_metrics:dropped_get(ActionId) + }, + ?assertEqual( + #{match => 1, dropped => 0, failed => 0, queuing => 0}, + Metrics + ). + +action_config(ConnectorName) -> + action_config(ConnectorName, _Overrides = #{}). + +action_config(ConnectorName, Overrides) -> + Cfg0 = action(ConnectorName), + emqx_utils_maps:deep_merge(Cfg0, Overrides). + +action(ConnectorName) -> + #{ + <<"description">> => <<"My elasticsearch test action">>, + <<"enable">> => true, + <<"parameters">> => #{ + <<"index">> => <<"${payload.index}">>, + <<"action">> => <<"create">>, + <<"doc">> => <<"${payload.doc}">>, + <<"overwrite">> => true + }, + <<"connector">> => ConnectorName, + <<"resource_opts">> => #{ + <<"health_check_interval">> => <<"30s">>, + <<"query_mode">> => <<"async">> + } + }. + +base_url(Config) -> + Host = ?config(es_host, Config), + Port = ?config(es_port, Config), + iolist_to_binary([ + "https://", + Host, + ":", + integer_to_binary(Port) + ]). + +connector_config(Config) -> + connector_config(_Overrides = #{}, Config). + +connector_config(Overrides, Config) -> + Defaults = + #{ + <<"base_url">> => base_url(Config), + <<"enable">> => true, + <<"authentication">> => #{ + <<"password">> => <<"emqx123">>, + <<"username">> => <<"elastic">> + }, + <<"description">> => <<"My elasticsearch test connector">>, + <<"connect_timeout">> => <<"15s">>, + <<"pool_size">> => 2, + <<"pool_type">> => <<"random">>, + <<"enable_pipelining">> => 100, + <<"ssl">> => #{ + <<"enable">> => true, + <<"hibernate_after">> => <<"5s">>, + <<"cacertfile">> => filename:join(?config(data_dir, Config), ?CA) + } + }, + emqx_utils_maps:deep_merge(Defaults, Overrides). + +create_connector(Name, Config) -> + Res = emqx_connector:create(?TYPE, Name, Config), + on_exit(fun() -> emqx_connector:remove(?TYPE, Name) end), + Res. + +create_action(Name, Config) -> + Res = emqx_bridge_v2:create(?TYPE, Name, Config), + on_exit(fun() -> emqx_bridge_v2:remove(?TYPE, Name) end), + Res. + +action_api_spec_props_for_get() -> + #{ + <<"bridge_elasticsearch.get_bridge_v2">> := + #{<<"properties">> := Props} + } = + emqx_bridge_v2_testlib:actions_api_spec_schemas(), + Props. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_create_remove_list(Config) -> + [] = emqx_bridge_v2:list(), + ConnectorConfig = connector_config(Config), + {ok, _} = emqx_connector:create(?TYPE, test_connector, ConnectorConfig), + ActionConfig = action(<<"test_connector">>), + {ok, _} = emqx_bridge_v2:create(?TYPE, test_action_1, ActionConfig), + [ActionInfo] = emqx_bridge_v2:list(), + #{ + name := <<"test_action_1">>, + type := <<"elasticsearch">>, + raw_config := _RawConfig + } = ActionInfo, + {ok, _} = emqx_bridge_v2:create(?TYPE, test_action_2, ActionConfig), + 2 = length(emqx_bridge_v2:list()), + ok = emqx_bridge_v2:remove(?TYPE, test_action_1), + 1 = length(emqx_bridge_v2:list()), + ok = emqx_bridge_v2:remove(?TYPE, test_action_2), + [] = emqx_bridge_v2:list(), + emqx_connector:remove(?TYPE, test_connector), + ok. + +%% Test sending a message to a bridge V2 +t_send_message(Config) -> + ConnectorConfig = connector_config(Config), + {ok, _} = emqx_connector:create(?TYPE, test_connector2, ConnectorConfig), + ActionConfig = action(<<"test_connector2">>), + {ok, _} = emqx_bridge_v2:create(?TYPE, test_action_1, ActionConfig), + %% Use the action to send a message + check_send_message_with_action(test_action_1, test_connector2), + %% Create a few more bridges with the same connector and test them + BridgeNames1 = [ + list_to_atom("test_bridge_v2_" ++ integer_to_list(I)) + || I <- lists:seq(2, 10) + ], + lists:foreach( + fun(BridgeName) -> + {ok, _} = emqx_bridge_v2:create(?TYPE, BridgeName, ActionConfig), + check_send_message_with_action(BridgeName, test_connector2) + end, + BridgeNames1 + ), + BridgeNames = [test_bridge_v2_1 | BridgeNames1], + %% Send more messages to the bridges + lists:foreach( + fun(BridgeName) -> + lists:foreach( + fun(_) -> + check_send_message_with_action(BridgeName, test_connector2) + end, + lists:seq(1, 10) + ) + end, + BridgeNames + ), + %% Remove all the bridges + lists:foreach( + fun(BridgeName) -> + ok = emqx_bridge_v2:remove(?TYPE, BridgeName) + end, + BridgeNames + ), + emqx_connector:remove(?TYPE, test_connector2), + ok. + +%% Test that we can get the status of the bridge V2 +t_health_check(Config) -> + BridgeV2Config = action(<<"test_connector3">>), + ConnectorConfig = connector_config(Config), + {ok, _} = emqx_connector:create(?TYPE, test_connector3, ConnectorConfig), + {ok, _} = emqx_bridge_v2:create(?TYPE, test_bridge_v2, BridgeV2Config), + #{status := connected} = emqx_bridge_v2:health_check(?TYPE, test_bridge_v2), + ok = emqx_bridge_v2:remove(?TYPE, test_bridge_v2), + %% Check behaviour when bridge does not exist + {error, bridge_not_found} = emqx_bridge_v2:health_check(?TYPE, test_bridge_v2), + ok = emqx_connector:remove(?TYPE, test_connector3), + ok. + +t_bad_url(Config) -> + ConnectorName = <<"test_connector">>, + ActionName = <<"test_action">>, + ActionConfig = action(<<"test_connector">>), + ConnectorConfig0 = connector_config(Config), + ConnectorConfig = ConnectorConfig0#{<<"base_url">> := <<"bad_host:9092">>}, + ?assertMatch({ok, _}, create_connector(ConnectorName, ConnectorConfig)), + ?assertMatch({ok, _}, create_action(ActionName, ActionConfig)), + ?assertMatch( + {ok, #{ + resource_data := + #{ + status := ?status_disconnected, + error := failed_to_start_elasticsearch_bridge + } + }}, + emqx_connector:lookup(?TYPE, ConnectorName) + ), + ?assertMatch({ok, #{status := ?status_disconnected}}, emqx_bridge_v2:lookup(?TYPE, ActionName)), + ok. + +t_parameters_key_api_spec(_Config) -> + ActionProps = action_api_spec_props_for_get(), + ?assertNot(is_map_key(<<"elasticsearch">>, ActionProps), #{action_props => ActionProps}), + ?assert(is_map_key(<<"parameters">>, ActionProps), #{action_props => ActionProps}), + ok. + +t_http_api_get(Config) -> + ConnectorName = <<"test_connector">>, + ActionName = <<"test_action">>, + ActionConfig = action(ConnectorName), + ConnectorConfig = connector_config(Config), + ?assertMatch({ok, _}, create_connector(ConnectorName, ConnectorConfig)), + ?assertMatch({ok, _}, create_action(ActionName, ActionConfig)), + ?assertMatch( + {ok, + {{_, 200, _}, _, [ + #{ + <<"connector">> := ConnectorName, + <<"description">> := <<"My elasticsearch test action">>, + <<"enable">> := true, + <<"error">> := <<>>, + <<"name">> := ActionName, + <<"node_status">> := + [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">>, + <<"status_reason">> := <<>> + } + ], + <<"parameters">> := + #{ + <<"action">> := <<"create">>, + <<"doc">> := <<"${payload.doc}">>, + <<"index">> := <<"${payload.index}">>, + <<"max_retries">> := 2, + <<"overwrite">> := true + }, + <<"resource_opts">> := #{<<"query_mode">> := <<"async">>}, + <<"status">> := <<"connected">>, + <<"status_reason">> := <<>>, + <<"type">> := <<"elasticsearch">> + } + ]}}, + emqx_bridge_v2_testlib:list_bridges_api() + ), + ok. diff --git a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE_data/es.crt b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE_data/es.crt new file mode 100644 index 000000000..2d47d555c --- /dev/null +++ b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE_data/es.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSjCCAjKgAwIBAgIVAIrN275DCtGnotTPpxwvQ5751N4OMA0GCSqGSIb3DQEB +CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu +ZXJhdGVkIENBMB4XDTI0MDExNjAyMzIyMFoXDTI3MDExNTAyMzIyMFowNDEyMDAG +A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew +ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCy0nwiEurUkIPFMLV1weVM +pPk/AlwZUzqjkeL44gsY53XI9Q05w/sL9u6PzwrXgTCFWNXzI9+MoAtp8phPkn14 +cmg5/3sLe9YcFVFjYK/MoljlUbPDj+4dgk8l+w5FRSi0+JN5krUm7rYk9lojAkeS +fX8RU7ekKGbjBXIFtPxX5GNadu9RidR5GkHM3XroAIoris8bFOzMgFn9iybYnkhq +0S+Hpv0A8FVxzle0KNbPpsIkxXH2DnP2iPTDym9xJNl9Iv9MPtj9XaamH7TmXcSt +MbjkAudKsCw4bRuhHonM16DIUr8sX5UcRcAWyJ1x1qpZaOzMdh2VdYAHNuOsZwzJ +AgMBAAGjUzBRMB0GA1UdDgQWBBTAyDlp8NZfPe8NCGVlHJSVclGOhTAfBgNVHSME +GDAWgBTAyDlp8NZfPe8NCGVlHJSVclGOhTAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBCwUAA4IBAQAeIUXRKmC53iirY4P49YspLafspAMf4ndMFQAp+Oc223Vs +hQC4axNoYnUdzWDH6LioAN7P826xNPqtXvTZF9fmeX7K8Nm9Kdj+for+QQI3j6+X +zq98VVkACb8b/Mc9Nac/WBbv/1IKyKgNNta7//WNPgAFolOfti/C0NLsPcKhrM9L +mGbvRX8ZjH8pVJ0YTy4/xfDcF7G/Lxl4Yvb0ZXpuQbvE1+Y0h5aoTNshT/skJxC4 +iyVseYr21s3pptKcr6H9KZuSdZe5pbEo+81nT15w+50aswFLk9GCYh5UsQ+1jkRK +cKgxP93i6x8BVbQJGKi1A1jhauSKX2IpWZQsHy4p +-----END CERTIFICATE----- diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index 27dfcba2a..92cf9439e 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -150,6 +150,9 @@ post_config_update([?ROOT_KEY, Type, Name], '$remove', _, _OldConf, _AppEnvs) -> ok = emqx_connector_resource:remove(Type, Name), ?tp(connector_post_config_update_done, #{}), ok; + {error, not_found} -> + ?tp(connector_post_config_update_done, #{}), + ok; {ok, Channels} -> {error, {active_channels, Channels}} end; diff --git a/rel/i18n/emqx_bridge_es.hocon b/rel/i18n/emqx_bridge_es.hocon index 62778a712..f5d0f3c02 100644 --- a/rel/i18n/emqx_bridge_es.hocon +++ b/rel/i18n/emqx_bridge_es.hocon @@ -97,7 +97,7 @@ config_parameters_require_alias.label: """_require_alias""" config_parameters_doc.desc: -"""JSON document""" +"""JSON document. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc.""" config_parameters_doc.label: """doc""" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 7959581a9..af04fd9ee 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -246,6 +246,9 @@ for dep in ${CT_DEPS}; do otel) FILES+=( '.ci/docker-compose-file/docker-compose-otel.yaml' ) ;; + elasticsearch) + FILES+=( '.ci/docker-compose-file/docker-compose-elastic-search-tls.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 59797cfea7a2f79da3d95e15c1279d8c0a9deafd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 18 Jan 2024 15:43:02 +0800 Subject: [PATCH 61/62] feat: es's update support doc_as_upsert --- apps/emqx_bridge_es/src/emqx_bridge_es.erl | 12 ++ .../src/emqx_bridge_es_connector.erl | 33 +++++- .../test/emqx_bridge_es_SUITE.erl | 109 ++++++++++-------- .../src/emqx_bridge_http_connector.erl | 42 ++++--- rel/i18n/emqx_bridge_es.hocon | 6 + 5 files changed, 129 insertions(+), 73 deletions(-) diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.erl b/apps/emqx_bridge_es/src/emqx_bridge_es.erl index 569b67dba..032439574 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.erl @@ -73,6 +73,7 @@ fields(action_update) -> index(), id(true), doc(), + doc_as_upsert(), routing(), require_alias() | http_common_opts() @@ -172,6 +173,17 @@ http_common_opts() -> emqx_bridge_http_schema:fields("parameters_opts") ). +doc_as_upsert() -> + {doc_as_upsert, + ?HOCON( + boolean(), + #{ + required => false, + default => false, + desc => ?DESC("config_doc_as_upsert") + } + )}. + routing() -> {routing, ?HOCON( diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 7e49aeb55..8b68af10f 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -33,6 +33,8 @@ connector_example_values/0 ]). +-export([render_template/2]). + %% emqx_connector_resource behaviour callbacks -export([connector_config/2]). @@ -286,8 +288,12 @@ on_add_channel( method => method(Parameter), body => get_body_template(Parameter) }, + ChannelConfig = #{ + parameters => Parameter1, + render_template_func => fun ?MODULE:render_template/2 + }, {ok, State} = emqx_bridge_http_connector:on_add_channel( - InstanceId, State0, ChannelId, #{parameters => Parameter1} + InstanceId, State0, ChannelId, ChannelConfig ), Channel = Parameter1, Channels2 = Channels#{ChannelId => Channel}, @@ -310,9 +316,23 @@ on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> {error, not_exists} end. +render_template(Template, Msg) -> + % Ignoring errors here, undefined bindings will be replaced with empty string. + Opts = #{var_trans => fun to_string/2}, + {String, _Errors} = emqx_template:render(Template, {emqx_jsonish, Msg}, Opts), + String. + %%-------------------------------------------------------------------- %% Internal Functions %%-------------------------------------------------------------------- + +to_string(Name, Value) -> + emqx_template:to_string(render_var(Name, Value)). +render_var(_, undefined) -> + % NOTE Any allowed but undefined binding will be replaced with empty string + <<>>; +render_var(_Name, Value) -> + Value. %% delete DELETE //_doc/<_id> path(#{action := delete, id := Id, index := Index} = Action) -> BasePath = ["/", Index, "/_doc/", Id], @@ -370,5 +390,12 @@ handle_response({ok, Code, Body}) -> handle_response({error, _} = Error) -> Error. -get_body_template(#{doc := Doc}) -> Doc; -get_body_template(_) -> undefined. +get_body_template(#{action := update, doc := Doc} = Template) -> + case maps:get(doc_as_upsert, Template, false) of + false -> <<"{\"doc\":", Doc/binary, "}">>; + true -> <<"{\"doc\":", Doc/binary, ",\"doc_as_upsert\": true}">> + end; +get_body_template(#{doc := Doc}) -> + Doc; +get_body_template(_) -> + undefined. diff --git a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl index e7e2dba28..a9ff70957 100644 --- a/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl +++ b/apps/emqx_bridge_es/test/emqx_bridge_es_SUITE.erl @@ -103,45 +103,46 @@ end_per_testcase(_TestCase, _Config) -> %% Helper fns %%------------------------------------------------------------------------------------- -check_send_message_with_action(ActionName, ConnectorName) -> - #{payload := _Payload} = send_message(ActionName), +check_send_message_with_action(Topic, ActionName, ConnectorName) -> + send_message(Topic), %% ###################################### %% Check if message is sent to es %% ###################################### + timer:sleep(500), check_action_metrics(ActionName, ConnectorName). -send_message(ActionName) -> - %% ###################################### - %% Create message - %% ###################################### - Time = erlang:unique_integer(), - BinTime = integer_to_binary(Time), - Payload = #{<<"name">> => <<"emqx">>, <<"release_time">> => BinTime}, +send_message(Topic) -> + Now = emqx_utils_calendar:now_to_rfc3339(microsecond), + Doc = #{<<"name">> => <<"emqx">>, <<"release_date">> => Now}, Index = <<"emqx-test-index">>, - Msg = #{ - clientid => BinTime, - payload => Payload, - timestamp => Time, - index => Index - }, - %% ###################################### - %% Send message - %% ###################################### - emqx_bridge_v2:send_message(?TYPE, ActionName, Msg, #{}), - #{payload => Payload}. + Payload = emqx_utils_json:encode(#{doc => Doc, index => Index}), + + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + {ok, Client} = emqtt:start_link([{clientid, ClientId}, {port, 1883}]), + {ok, _} = emqtt:connect(Client), + ok = emqtt:publish(Client, Topic, Payload, [{qos, 0}]), + ok. check_action_metrics(ActionName, ConnectorName) -> ActionId = emqx_bridge_v2:id(?TYPE, ActionName, ConnectorName), Metrics = #{ match => emqx_resource_metrics:matched_get(ActionId), + success => emqx_resource_metrics:success_get(ActionId), failed => emqx_resource_metrics:failed_get(ActionId), queuing => emqx_resource_metrics:queuing_get(ActionId), dropped => emqx_resource_metrics:dropped_get(ActionId) }, ?assertEqual( - #{match => 1, dropped => 0, failed => 0, queuing => 0}, - Metrics + #{ + match => 1, + success => 1, + dropped => 0, + failed => 0, + queuing => 0 + }, + Metrics, + {ActionName, ConnectorName, ActionId} ). action_config(ConnectorName) -> @@ -164,7 +165,7 @@ action(ConnectorName) -> <<"connector">> => ConnectorName, <<"resource_opts">> => #{ <<"health_check_interval">> => <<"30s">>, - <<"query_mode">> => <<"async">> + <<"query_mode">> => <<"sync">> } }. @@ -235,7 +236,8 @@ t_create_remove_list(Config) -> #{ name := <<"test_action_1">>, type := <<"elasticsearch">>, - raw_config := _RawConfig + raw_config := _, + status := connected } = ActionInfo, {ok, _} = emqx_bridge_v2:create(?TYPE, test_action_2, ActionConfig), 2 = length(emqx_bridge_v2:list()), @@ -252,39 +254,44 @@ t_send_message(Config) -> {ok, _} = emqx_connector:create(?TYPE, test_connector2, ConnectorConfig), ActionConfig = action(<<"test_connector2">>), {ok, _} = emqx_bridge_v2:create(?TYPE, test_action_1, ActionConfig), + Rule = #{ + id => <<"rule:t_es">>, + sql => <<"SELECT\n *\nFROM\n \"es/#\"">>, + actions => [<<"elasticsearch:test_action_1">>], + description => <<"sink doc to elasticsearch">> + }, + {ok, _} = emqx_rule_engine:create_rule(Rule), %% Use the action to send a message - check_send_message_with_action(test_action_1, test_connector2), + check_send_message_with_action(<<"es/1">>, test_action_1, test_connector2), %% Create a few more bridges with the same connector and test them - BridgeNames1 = [ - list_to_atom("test_bridge_v2_" ++ integer_to_list(I)) - || I <- lists:seq(2, 10) - ], - lists:foreach( - fun(BridgeName) -> - {ok, _} = emqx_bridge_v2:create(?TYPE, BridgeName, ActionConfig), - check_send_message_with_action(BridgeName, test_connector2) - end, - BridgeNames1 - ), - BridgeNames = [test_bridge_v2_1 | BridgeNames1], - %% Send more messages to the bridges - lists:foreach( - fun(BridgeName) -> - lists:foreach( - fun(_) -> - check_send_message_with_action(BridgeName, test_connector2) - end, - lists:seq(1, 10) - ) - end, - BridgeNames - ), + ActionNames1 = + lists:foldl( + fun(I, Acc) -> + Seq = integer_to_binary(I), + ActionNameStr = "test_action_" ++ integer_to_list(I), + ActionName = list_to_atom(ActionNameStr), + {ok, _} = emqx_bridge_v2:create(?TYPE, ActionName, ActionConfig), + Rule1 = #{ + id => <<"rule:t_es", Seq/binary>>, + sql => <<"SELECT\n *\nFROM\n \"es/", Seq/binary, "\"">>, + actions => [<<"elasticsearch:", (list_to_binary(ActionNameStr))/binary>>], + description => <<"sink doc to elasticsearch">> + }, + {ok, _} = emqx_rule_engine:create_rule(Rule1), + Topic = <<"es/", Seq/binary>>, + check_send_message_with_action(Topic, ActionName, test_connector2), + [ActionName | Acc] + end, + [], + lists:seq(2, 10) + ), + ActionNames = [test_action_1 | ActionNames1], %% Remove all the bridges lists:foreach( fun(BridgeName) -> ok = emqx_bridge_v2:remove(?TYPE, BridgeName) end, - BridgeNames + ActionNames ), emqx_connector:remove(?TYPE, test_connector2), ok. @@ -361,7 +368,7 @@ t_http_api_get(Config) -> <<"max_retries">> := 2, <<"overwrite">> := true }, - <<"resource_opts">> := #{<<"query_mode">> := <<"async">>}, + <<"resource_opts">> := #{<<"query_mode">> := <<"sync">>}, <<"status">> := <<"connected">>, <<"status_reason">> := <<>>, <<"type">> := <<"elasticsearch">> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 8f54694e9..81acec602 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -266,7 +266,9 @@ on_add_channel( ) -> InstalledActions = maps:get(installed_actions, OldState, #{}), {ok, ActionState} = do_create_http_action(ActionConfig), - NewInstalledActions = maps:put(ActionId, ActionState, InstalledActions), + RenderTemplate = maps:get(render_template_func, ActionConfig, fun render_template/2), + ActionState1 = ActionState#{render_template_func => RenderTemplate}, + NewInstalledActions = maps:put(ActionId, ActionState1, InstalledActions), NewState = maps:put(installed_actions, NewInstalledActions, OldState), {ok, NewState}. @@ -631,9 +633,10 @@ parse_template(String) -> process_request_and_action(Request, ActionState, Msg) -> MethodTemplate = maps:get(method, ActionState), - Method = make_method(render_template_string(MethodTemplate, Msg)), - PathPrefix = unicode:characters_to_list(render_template(maps:get(path, Request), Msg)), - PathSuffix = unicode:characters_to_list(render_template(maps:get(path, ActionState), Msg)), + RenderTmplFunc = maps:get(render_template_func, ActionState), + Method = make_method(render_template_string(MethodTemplate, RenderTmplFunc, Msg)), + PathPrefix = unicode:characters_to_list(RenderTmplFunc(maps:get(path, Request), Msg)), + PathSuffix = unicode:characters_to_list(RenderTmplFunc(maps:get(path, ActionState), Msg)), Path = case PathSuffix of @@ -644,11 +647,11 @@ process_request_and_action(Request, ActionState, Msg) -> HeadersTemplate1 = maps:get(headers, Request), HeadersTemplate2 = maps:get(headers, ActionState), Headers = merge_proplist( - render_headers(HeadersTemplate1, Msg), - render_headers(HeadersTemplate2, Msg) + render_headers(HeadersTemplate1, RenderTmplFunc, Msg), + render_headers(HeadersTemplate2, RenderTmplFunc, Msg) ), BodyTemplate = maps:get(body, ActionState), - Body = render_request_body(BodyTemplate, Msg), + Body = render_request_body(BodyTemplate, RenderTmplFunc, Msg), #{ method => Method, path => Path, @@ -681,25 +684,26 @@ process_request( } = Conf, Msg ) -> + RenderTemplateFun = fun render_template/2, Conf#{ - method => make_method(render_template_string(MethodTemplate, Msg)), - path => unicode:characters_to_list(render_template(PathTemplate, Msg)), - body => render_request_body(BodyTemplate, Msg), - headers => render_headers(HeadersTemplate, Msg), + method => make_method(render_template_string(MethodTemplate, RenderTemplateFun, Msg)), + path => unicode:characters_to_list(RenderTemplateFun(PathTemplate, Msg)), + body => render_request_body(BodyTemplate, RenderTemplateFun, Msg), + headers => render_headers(HeadersTemplate, RenderTemplateFun, Msg), request_timeout => ReqTimeout }. -render_request_body(undefined, Msg) -> +render_request_body(undefined, _, Msg) -> emqx_utils_json:encode(Msg); -render_request_body(BodyTks, Msg) -> - render_template(BodyTks, Msg). +render_request_body(BodyTks, RenderTmplFunc, Msg) -> + RenderTmplFunc(BodyTks, Msg). -render_headers(HeaderTks, Msg) -> +render_headers(HeaderTks, RenderTmplFunc, Msg) -> lists:map( fun({K, V}) -> { - render_template_string(K, Msg), - render_template_string(emqx_secret:unwrap(V), Msg) + render_template_string(K, RenderTmplFunc, Msg), + render_template_string(emqx_secret:unwrap(V), RenderTmplFunc, Msg) } end, HeaderTks @@ -710,8 +714,8 @@ render_template(Template, Msg) -> {String, _Errors} = emqx_template:render(Template, {emqx_jsonish, Msg}), String. -render_template_string(Template, Msg) -> - unicode:characters_to_binary(render_template(Template, Msg)). +render_template_string(Template, RenderTmplFunc, Msg) -> + unicode:characters_to_binary(RenderTmplFunc(Template, Msg)). make_method(M) when M == <<"POST">>; M == <<"post">> -> post; make_method(M) when M == <<"PUT">>; M == <<"put">> -> put; diff --git a/rel/i18n/emqx_bridge_es.hocon b/rel/i18n/emqx_bridge_es.hocon index f5d0f3c02..8ad11f05b 100644 --- a/rel/i18n/emqx_bridge_es.hocon +++ b/rel/i18n/emqx_bridge_es.hocon @@ -56,6 +56,12 @@ config_routing.desc: config_routing.label: """Routing""" +config_doc_as_upsert.desc: +"""Instead of sending a partial doc plus an upsert doc, +you can set doc_as_upsert to true to use the contents of doc as the upsert value.""" +config_doc_as_upsert.label: +"""doc_as_upsert""" + config_wait_for_active_shards.desc: """The number of shard copies that must be active before proceeding with the operation. Set to all or any positive integer up to the total number of shards in the index (number_of_replicas+1). From dae835635cacdb7967a6d64a6b2b48a89d629e2c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 18 Jan 2024 16:23:02 +0800 Subject: [PATCH 62/62] fix: don't crash in http SUITE --- apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl | 5 +++-- apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl | 2 +- changes/feat-12348.en.md | 1 + scripts/spellcheck/dicts/emqx.txt | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 changes/feat-12348.en.md diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 81acec602..a148a4d16 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -38,6 +38,7 @@ ]). -export([reply_delegator/3]). +-export([render_template/2]). -export([ roots/0, @@ -266,8 +267,8 @@ on_add_channel( ) -> InstalledActions = maps:get(installed_actions, OldState, #{}), {ok, ActionState} = do_create_http_action(ActionConfig), - RenderTemplate = maps:get(render_template_func, ActionConfig, fun render_template/2), - ActionState1 = ActionState#{render_template_func => RenderTemplate}, + RenderTmplFunc = maps:get(render_template_func, ActionConfig, fun ?MODULE:render_template/2), + ActionState1 = ActionState#{render_template_func => RenderTmplFunc}, NewInstalledActions = maps:put(ActionId, ActionState1, InstalledActions), NewState = maps:put(installed_actions, NewInstalledActions, OldState), {ok, NewState}. diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index 3b7303300..932191ec5 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -146,7 +146,7 @@ end_per_testcase(_TestCase, Config) -> %%------------------------------------------------------------------------------ %% HTTP server for testing -%% (Orginally copied from emqx_bridge_api_SUITE) +%% (Originally copied from emqx_bridge_api_SUITE) %%------------------------------------------------------------------------------ start_http_server(HTTPServerConfig) -> process_flag(trap_exit, true), diff --git a/changes/feat-12348.en.md b/changes/feat-12348.en.md new file mode 100644 index 000000000..cf84bf5e2 --- /dev/null +++ b/changes/feat-12348.en.md @@ -0,0 +1 @@ +Added support for Elasticsearch Bridge. diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 1d98d82db..c2f5f54ef 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -299,3 +299,5 @@ now_us ns elasticsearch ElasticSearch +doc_as_upsert +upsert