diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d8c90965b..74f06969a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,8 @@ Fixes - +Release version: v/e5.? + +## Summary ## PR Checklist Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b23f91128..4a0d0403f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,14 @@ jobs: upload: runs-on: ubuntu-22.04 permissions: + contents: write + checks: write packages: write + actions: read + issues: read + pull-requests: read + repository-projects: read + statuses: read strategy: fail-fast: false steps: @@ -45,11 +52,13 @@ jobs: v*) echo "profile=emqx" >> $GITHUB_OUTPUT echo "version=$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT + echo "ref_name=v$(./pkg-vsn.sh emqx)" >> "$GITHUB_ENV" echo "s3dir=emqx-ce" >> $GITHUB_OUTPUT ;; e*) echo "profile=emqx-enterprise" >> $GITHUB_OUTPUT echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT + echo "ref_name=e$(./pkg-vsn.sh emqx-enterprise)" >> "$GITHUB_ENV" echo "s3dir=emqx-ee" >> $GITHUB_OUTPUT ;; esac @@ -57,14 +66,15 @@ jobs: run: | BUCKET=${{ secrets.AWS_S3_BUCKET }} OUTPUT_DIR=${{ steps.profile.outputs.s3dir }} - aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages - - uses: alexellis/upload-assets@0.4.0 + aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ env.ref_name }} packages + - uses: emqx/upload-assets@8d2083b4dbe3151b0b735572eaa153b6acb647fe # 0.5.0 env: GITHUB_TOKEN: ${{ github.token }} with: asset_paths: '["packages/*"]' + tag_name: "${{ env.ref_name }}" - name: update to emqx.io - if: startsWith(github.ref_name, 'v') && ((github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts) + if: startsWith(env.ref_name, 'v') && ((github.event_name == 'release' && !github.event.release.prerelease) || inputs.publish_release_artefacts) run: | set -eux curl -w %{http_code} \ @@ -72,10 +82,10 @@ jobs: -H "Content-Type: application/json" \ -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \ -X POST \ - -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ + -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: Push to packagecloud.io - if: (github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts + if: (github.event_name == 'release' && !github.event.release.prerelease) || inputs.publish_release_artefacts env: PROFILE: ${{ steps.profile.outputs.profile }} VERSION: ${{ steps.profile.outputs.version }} diff --git a/.github/workflows/upload-helm-charts.yaml b/.github/workflows/upload-helm-charts.yaml index 593a78a7c..44261d137 100644 --- a/.github/workflows/upload-helm-charts.yaml +++ b/.github/workflows/upload-helm-charts.yaml @@ -43,7 +43,7 @@ jobs: ;; esac - uses: emqx/push-helm-action@v1.1 - if: github.event_name == 'release' && !github.event.prerelease + if: github.event_name == 'release' && !github.event.release.prerelease with: charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}" version: ${{ steps.profile.outputs.version }} diff --git a/.tool-versions b/.tool-versions index a988325fa..824207a4a 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 25.3.2-2 -elixir 1.14.5-otp-25 +erlang 26.1.2-1 +elixir 1.15.7-otp-26 diff --git a/Makefile b/Makefile index 7c9638dd5..41812f6d9 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.5.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.3.2-beta.1 +export EMQX_DASHBOARD_VERSION ?= v1.5.2 +export EMQX_EE_DASHBOARD_VERSION ?= e1.3.2 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 2f9254d70..299486ad1 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.3.2"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.3.2-alpha.1"). +-define(EMQX_RELEASE_EE, "5.3.2"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl index 56246e743..5e1297df6 100644 --- a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl @@ -9,6 +9,7 @@ -include_lib("stdlib/include/assert.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -39,7 +40,7 @@ init_per_testcase(TestCase, Config) when Cluster = cluster(#{n => 1}), ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}, NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts), - Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts), + Nodes = emqx_cth_cluster:start(NodeSpecs), [ {cluster, Cluster}, {node_specs, NodeSpecs}, @@ -47,12 +48,36 @@ init_per_testcase(TestCase, Config) when {nodes, Nodes} | Config ]; +init_per_testcase(t_session_gc = TestCase, Config) -> + Opts = #{ + n => 3, + roles => [core, core, replicant], + extra_emqx_conf => + "\n session_persistence {" + "\n last_alive_update_interval = 500ms " + "\n session_gc_interval = 2s " + "\n session_gc_batch_size = 1 " + "\n }" + }, + Cluster = cluster(Opts), + ClusterOpts = #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}, + NodeSpecs = emqx_cth_cluster:mk_nodespecs(Cluster, ClusterOpts), + Nodes = emqx_cth_cluster:start(Cluster, ClusterOpts), + [ + {cluster, Cluster}, + {node_specs, NodeSpecs}, + {cluster_opts, ClusterOpts}, + {nodes, Nodes}, + {gc_interval, timer:seconds(2)} + | Config + ]; init_per_testcase(_TestCase, Config) -> Config. end_per_testcase(TestCase, Config) when TestCase =:= t_session_subscription_idempotency; - TestCase =:= t_session_unsubscription_idempotency + TestCase =:= t_session_unsubscription_idempotency; + TestCase =:= t_session_gc -> Nodes = ?config(nodes, Config), emqx_common_test_helpers:call_janitor(60_000), @@ -66,20 +91,32 @@ end_per_testcase(_TestCase, _Config) -> %% Helper fns %%------------------------------------------------------------------------------ -cluster(#{n := N}) -> - Spec = #{role => core, apps => app_specs()}, +cluster(#{n := N} = Opts) -> + MkRole = fun(M) -> + case maps:get(roles, Opts, undefined) of + undefined -> + core; + Roles -> + lists:nth(M, Roles) + end + end, + MkSpec = fun(M) -> #{role => MkRole(M), apps => app_specs(Opts)} end, lists:map( fun(M) -> Name = list_to_atom("ds_SUITE" ++ integer_to_list(M)), - {Name, Spec} + {Name, MkSpec(M)} end, lists:seq(1, N) ). app_specs() -> + app_specs(_Opts = #{}). + +app_specs(Opts) -> + ExtraEMQXConf = maps:get(extra_emqx_conf, Opts, ""), [ emqx_durable_storage, - {emqx, "session_persistence = {enable = true}"} + {emqx, "session_persistence = {enable = true}" ++ ExtraEMQXConf} ]. get_mqtt_port(Node, Type) -> @@ -115,33 +152,37 @@ start_client(Opts0 = #{}) -> restart_node(Node, NodeSpec) -> ?tp(will_restart_node, #{}), - ?tp(notice, "restarting node", #{node => Node}), - true = monitor_node(Node, true), - ok = erpc:call(Node, init, restart, []), - receive - {nodedown, Node} -> - ok - after 10_000 -> - ct:fail("node ~p didn't stop", [Node]) - end, - ?tp(notice, "waiting for nodeup", #{node => Node}), + emqx_cth_cluster:restart(Node, NodeSpec), wait_nodeup(Node), - wait_gen_rpc_down(NodeSpec), - ?tp(notice, "restarting apps", #{node => Node}), - Apps = maps:get(apps, NodeSpec), - ok = erpc:call(Node, emqx_cth_suite, load_apps, [Apps]), - _ = erpc:call(Node, emqx_cth_suite, start_apps, [Apps, NodeSpec]), - %% have to re-inject this so that we may stop the node succesfully at the - %% end.... - ok = emqx_cth_cluster:set_node_opts(Node, NodeSpec), - ok = snabbkaffe:forward_trace(Node), - ?tp(notice, "node restarted", #{node => Node}), ?tp(restarted_node, #{}), ok. is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}}) -> EI > 0. +list_all_sessions(Node) -> + erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []). + +list_all_subscriptions(Node) -> + erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []). + +list_all_pubranges(Node) -> + erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []). + +prop_only_cores_run_gc(CoreNodes) -> + {"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}. +prop_only_cores_run_gc(Trace, CoreNodes) -> + GCNodes = lists:usort([ + N + || #{ + ?snk_kind := K, + ?snk_meta := #{node := N} + } <- Trace, + lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]), + N =/= node() + ]), + ?assertEqual(lists:usort(CoreNodes), GCNodes). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -221,9 +262,10 @@ t_session_subscription_idempotency(Config) -> end, fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), + ConnInfo = #{}, ?assertMatch( #{subscriptions := #{SubTopicFilter := #{}}}, - erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId]) + erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo]) ) end ), @@ -294,9 +336,10 @@ t_session_unsubscription_idempotency(Config) -> end, fun(Trace) -> ct:pal("trace:\n ~p", [Trace]), + ConnInfo = #{}, ?assertMatch( #{subscriptions := Subs = #{}} when map_size(Subs) =:= 0, - erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId]) + erpc:call(Node1, emqx_persistent_session_ds, session_open, [ClientId, ConnInfo]) ), ok end @@ -387,3 +430,201 @@ do_t_session_discard(Params) -> end ), ok. + +t_session_expiration1(Config) -> + ClientId = atom_to_binary(?FUNCTION_NAME), + Opts = #{ + clientid => ClientId, + sequence => [ + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}, + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 1}}, #{}}, + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}} + ] + }, + do_t_session_expiration(Config, Opts). + +t_session_expiration2(Config) -> + ClientId = atom_to_binary(?FUNCTION_NAME), + Opts = #{ + clientid => ClientId, + sequence => [ + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}}, + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{ + 'Session-Expiry-Interval' => 1 + }}, + {#{clean_start => false, properties => #{'Session-Expiry-Interval' => 30}}, #{}} + ] + }, + do_t_session_expiration(Config, Opts). + +do_t_session_expiration(_Config, Opts) -> + #{ + clientid := ClientId, + sequence := [ + {FirstConn, FirstDisconn}, + {SecondConn, SecondDisconn}, + {ThirdConn, ThirdDisconn} + ] + } = Opts, + CommonParams = #{proto_ver => v5, clientid => ClientId}, + ?check_trace( + begin + Topic = <<"some/topic">>, + Params0 = maps:merge(CommonParams, FirstConn), + Client0 = start_client(Params0), + {ok, _} = emqtt:connect(Client0), + {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2), + Subs0 = emqx_persistent_session_ds:list_all_subscriptions(), + ?assertEqual(1, map_size(Subs0), #{subs => Subs0}), + Info0 = maps:from_list(emqtt:info(Client0)), + ?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}), + emqtt:disconnect(Client0, ?RC_NORMAL_DISCONNECTION, FirstDisconn), + + Params1 = maps:merge(CommonParams, SecondConn), + Client1 = start_client(Params1), + {ok, _} = emqtt:connect(Client1), + Info1 = maps:from_list(emqtt:info(Client1)), + ?assertEqual(1, maps:get(session_present, Info1), #{info => Info1}), + Subs1 = emqtt:subscriptions(Client1), + ?assertEqual([], Subs1), + emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn), + + ct:sleep(1_500), + + Params2 = maps:merge(CommonParams, ThirdConn), + Client2 = start_client(Params2), + {ok, _} = emqtt:connect(Client2), + Info2 = maps:from_list(emqtt:info(Client2)), + ?assertEqual(0, maps:get(session_present, Info2), #{info => Info2}), + Subs2 = emqtt:subscriptions(Client2), + ?assertEqual([], Subs2), + emqtt:publish(Client2, Topic, <<"payload">>), + ?assertNotReceive({publish, #{topic := Topic}}), + %% ensure subscriptions are absent from table. + ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()), + emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn), + + ok + end, + [] + ), + ok. + +t_session_gc(Config) -> + GCInterval = ?config(gc_interval, Config), + [Node1, Node2, Node3] = Nodes = ?config(nodes, Config), + CoreNodes = [Node1, Node2], + [ + Port1, + Port2, + Port3 + ] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes), + CommonParams = #{ + clean_start => false, + proto_ver => v5 + }, + StartClient = fun(ClientId, Port, ExpiryInterval) -> + Params = maps:merge(CommonParams, #{ + clientid => ClientId, + port => Port, + properties => #{'Session-Expiry-Interval' => ExpiryInterval} + }), + Client = start_client(Params), + {ok, _} = emqtt:connect(Client), + Client + end, + + ?check_trace( + begin + ClientId0 = <<"session_gc0">>, + Client0 = StartClient(ClientId0, Port1, 30), + + ClientId1 = <<"session_gc1">>, + Client1 = StartClient(ClientId1, Port2, 1), + + ClientId2 = <<"session_gc2">>, + Client2 = StartClient(ClientId2, Port3, 1), + + lists:foreach( + fun(Client) -> + Topic = <<"some/topic">>, + Payload = <<"hi">>, + {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client, Topic, ?QOS_1), + {ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1), + ok + end, + [Client0, Client1, Client2] + ), + + %% Clients are still alive; no session is garbage collected. + Res0 = ?block_until( + #{ + ?snk_kind := ds_session_gc, + ?snk_span := {complete, _}, + ?snk_meta := #{node := N} + } when + N =/= node(), + 3 * GCInterval + 1_000 + ), + ?assertMatch({ok, _}, Res0), + {ok, #{?snk_meta := #{time := T0}}} = Res0, + Sessions0 = list_all_sessions(Node1), + Subs0 = list_all_subscriptions(Node1), + ?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}), + ?assertEqual(3, map_size(Subs0), #{subs => Subs0}), + + %% Now we disconnect 2 of them; only those should be GC'ed. + ?assertMatch( + {ok, {ok, _}}, + ?wait_async_action( + emqtt:stop(Client1), + #{?snk_kind := terminate}, + 1_000 + ) + ), + ct:pal("disconnected client1"), + ?assertMatch( + {ok, {ok, _}}, + ?wait_async_action( + emqtt:stop(Client2), + #{?snk_kind := terminate}, + 1_000 + ) + ), + ct:pal("disconnected client2"), + ?assertMatch( + {ok, _}, + ?block_until( + #{ + ?snk_kind := ds_session_gc_cleaned, + ?snk_meta := #{node := N, time := T}, + session_ids := [ClientId1] + } when + N =/= node() andalso T > T0, + 4 * GCInterval + 1_000 + ) + ), + ?assertMatch( + {ok, _}, + ?block_until( + #{ + ?snk_kind := ds_session_gc_cleaned, + ?snk_meta := #{node := N, time := T}, + session_ids := [ClientId2] + } when + N =/= node() andalso T > T0, + 4 * GCInterval + 1_000 + ) + ), + Sessions1 = list_all_sessions(Node1), + Subs1 = list_all_subscriptions(Node1), + ?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}), + ?assertEqual(1, map_size(Subs1), #{subs => Subs1}), + + ok + end, + [ + prop_only_cores_run_gc(CoreNodes) + ] + ), + ok. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 5400cda2f..7d160e9cf 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,9 +27,9 @@ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, - {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, - {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}}, + {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 0545f36a5..915a66f17 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.14"}, + {vsn, "5.1.15"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 306341700..816ab7b2b 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -423,6 +423,7 @@ handle_in( {ok, Channel} end; handle_in( + %% TODO: Why discard the Reason Code? ?PUBREC_PACKET(PacketId, _ReasonCode, Properties), Channel = #channel{clientinfo = ClientInfo, session = Session} @@ -1204,12 +1205,13 @@ handle_info( #channel{ conn_state = ConnState, clientinfo = ClientInfo, + conninfo = ConnInfo, session = Session } ) when ConnState =:= connected orelse ConnState =:= reauthenticating -> - {Intent, Session1} = emqx_session:disconnect(ClientInfo, Session), + {Intent, Session1} = emqx_session:disconnect(ClientInfo, ConnInfo, Session), Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)), Channel2 = Channel1#channel{session = Session1}, case maybe_shutdown(Reason, Intent, Channel2) of @@ -1321,7 +1323,8 @@ handle_timeout( {ok, Replies, NSession} -> handle_out(publish, Replies, Channel#channel{session = NSession}) end; -handle_timeout(_TRef, expire_session, Channel) -> +handle_timeout(_TRef, expire_session, Channel = #channel{session = Session}) -> + ok = emqx_session:destroy(Session), shutdown(expired, Channel); handle_timeout( _TRef, diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 537c60876..92b95c7c3 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -91,6 +91,7 @@ clean_down/1, mark_channel_connected/1, mark_channel_disconnected/1, + is_channel_connected/1, get_connected_client_count/0 ]). diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index 9db73e8e4..e7420b4da 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -47,7 +47,17 @@ init([]) -> Locker = child_spec(emqx_cm_locker, 5000, worker), Registry = child_spec(emqx_cm_registry, 5000, worker), Manager = child_spec(emqx_cm, 5000, worker), - {ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}. + DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor), + Children = + [ + Banned, + Flapping, + Locker, + Registry, + Manager, + DSSessionGCSup + ], + {ok, {SupFlags, Children}}. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx/src/emqx_persistent_message_ds_replayer.erl b/apps/emqx/src/emqx_persistent_message_ds_replayer.erl index d622444e9..fb8170904 100644 --- a/apps/emqx/src/emqx_persistent_message_ds_replayer.erl +++ b/apps/emqx/src/emqx_persistent_message_ds_replayer.erl @@ -19,7 +19,13 @@ -module(emqx_persistent_message_ds_replayer). %% API: --export([new/0, open/1, next_packet_id/1, replay/1, commit_offset/3, poll/3, n_inflight/1]). +-export([new/0, open/1, next_packet_id/1, n_inflight/1]). + +-export([poll/4, replay/2, commit_offset/4]). + +-export([seqno_to_packet_id/1, packet_id_to_seqno/2]). + +-export([committed_until/2]). %% internal exports: -export([]). @@ -27,7 +33,6 @@ -export_type([inflight/0, seqno/0]). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_utils/include/emqx_message.hrl"). -include("emqx_persistent_session_ds.hrl"). -ifdef(TEST). @@ -35,6 +40,13 @@ -include_lib("eunit/include/eunit.hrl"). -endif. +-define(EPOCH_SIZE, 16#10000). + +-define(ACK, 0). +-define(COMP, 1). + +-define(TRACK_FLAG(WHICH), (1 bsl WHICH)). + %%================================================================================ %% Type declarations %%================================================================================ @@ -42,15 +54,23 @@ %% Note: sequence numbers are monotonic; they don't wrap around: -type seqno() :: non_neg_integer(). +-type track() :: ack | comp. +-type commit_type() :: rec. + -record(inflight, { next_seqno = 1 :: seqno(), - acked_until = 1 :: seqno(), + commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()}, %% Ranges are sorted in ascending order of their sequence numbers. offset_ranges = [] :: [ds_pubrange()] }). -opaque inflight() :: #inflight{}. +-type reply_fun() :: fun( + (seqno(), emqx_types:message()) -> + emqx_session:replies() | {_AdvanceSeqno :: false, emqx_session:replies()} +). + %%================================================================================ %% API funcions %%================================================================================ @@ -61,10 +81,12 @@ new() -> -spec open(emqx_persistent_session_ds:id()) -> inflight(). open(SessionId) -> - Ranges = ro_transaction(fun() -> get_ranges(SessionId) end), - {AckedUntil, NextSeqno} = compute_inflight_range(Ranges), + {Ranges, RecUntil} = ro_transaction( + fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end + ), + {Commits, NextSeqno} = compute_inflight_range(Ranges), #inflight{ - acked_until = AckedUntil, + commits = Commits#{rec => RecUntil}, next_seqno = NextSeqno, offset_ranges = Ranges }. @@ -75,15 +97,30 @@ next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) -> {seqno_to_packet_id(LastSeqno), Inflight}. -spec n_inflight(inflight()) -> non_neg_integer(). -n_inflight(#inflight{next_seqno = NextSeqno, acked_until = AckedUntil}) -> - range_size(AckedUntil, NextSeqno). +n_inflight(#inflight{offset_ranges = Ranges}) -> + %% TODO + %% This is not very efficient. Instead, we can take the maximum of + %% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`. + %% This won't be exact number but a pessimistic estimate, but this way we + %% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2 + %% messages for some reason. For that to work, we need to additionally track + %% actual `AckedUntil` / `CompUntil` during `commit_offset/4`. + lists:foldl( + fun + (#ds_pubrange{type = ?T_CHECKPOINT}, N) -> + N; + (#ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until}, N) -> + N + range_size(First, Until) + end, + 0, + Ranges + ). --spec replay(inflight()) -> - {emqx_session:replies(), inflight()}. -replay(Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0}) -> +-spec replay(reply_fun(), inflight()) -> {emqx_session:replies(), inflight()}. +replay(ReplyFun, Inflight0 = #inflight{offset_ranges = Ranges0}) -> {Ranges, Replies} = lists:mapfoldr( fun(Range, Acc) -> - replay_range(Range, AckedUntil, Acc) + replay_range(ReplyFun, Range, Acc) end, [], Ranges0 @@ -91,43 +128,49 @@ replay(Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0}) Inflight = Inflight0#inflight{offset_ranges = Ranges}, {Replies, Inflight}. --spec commit_offset(emqx_persistent_session_ds:id(), emqx_types:packet_id(), inflight()) -> - {_IsValidOffset :: boolean(), inflight()}. +-spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) -> + {_IsValidOffset :: boolean(), inflight()} +when + Offset :: track() | commit_type(). commit_offset( SessionId, + Track, PacketId, - Inflight0 = #inflight{ - acked_until = AckedUntil, next_seqno = NextSeqno - } -) -> - case packet_id_to_seqno(NextSeqno, PacketId) of - Seqno when Seqno >= AckedUntil andalso Seqno < NextSeqno -> + Inflight0 = #inflight{commits = Commits} +) when Track == ack orelse Track == comp -> + case validate_commit(Track, PacketId, Inflight0) of + CommitUntil when is_integer(CommitUntil) -> %% TODO - %% We do not preserve `acked_until` in the database. Instead, we discard + %% We do not preserve `CommitUntil` in the database. Instead, we discard %% fully acked ranges from the database. In effect, this means that the - %% most recent `acked_until` the client has sent may be lost in case of a + %% most recent `CommitUntil` the client has sent may be lost in case of a %% crash or client loss. - Inflight1 = Inflight0#inflight{acked_until = next_seqno(Seqno)}, - Inflight = discard_acked(SessionId, Inflight1), + Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}}, + Inflight = discard_committed(SessionId, Inflight1), {true, Inflight}; - OutOfRange -> - ?SLOG(warning, #{ - msg => "out-of-order_ack", - acked_until => AckedUntil, - acked_seqno => OutOfRange, - next_seqno => NextSeqno, - packet_id => PacketId - }), + false -> + {false, Inflight0} + end; +commit_offset( + SessionId, + CommitType = rec, + PacketId, + Inflight0 = #inflight{commits = Commits} +) -> + case validate_commit(CommitType, PacketId, Inflight0) of + CommitUntil when is_integer(CommitUntil) -> + update_committed_offset(SessionId, CommitType, CommitUntil), + Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}}, + {true, Inflight}; + false -> {false, Inflight0} end. --spec poll(emqx_persistent_session_ds:id(), inflight(), pos_integer()) -> +-spec poll(reply_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) -> {emqx_session:replies(), inflight()}. -poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff -> - #inflight{next_seqno = NextSeqNo0, acked_until = AckedSeqno} = - Inflight0, +poll(ReplyFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE -> FetchThreshold = max(1, WindowSize div 2), - FreeSpace = AckedSeqno + WindowSize - NextSeqNo0, + FreeSpace = WindowSize - n_inflight(Inflight0), case FreeSpace >= FetchThreshold of false -> %% TODO: this branch is meant to avoid fetching data from @@ -138,9 +181,25 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff true -> %% TODO: Wrap this in `mria:async_dirty/2`? Streams = shuffle(get_streams(SessionId)), - fetch(SessionId, Inflight0, Streams, FreeSpace, []) + fetch(ReplyFun, SessionId, Inflight0, Streams, FreeSpace, []) end. +%% Which seqno this track is committed until. +%% "Until" means this is first seqno that is _not yet committed_ for this track. +-spec committed_until(track() | commit_type(), inflight()) -> seqno(). +committed_until(Track, #inflight{commits = Commits}) -> + maps:get(Track, Commits). + +-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0. +seqno_to_packet_id(Seqno) -> + Seqno rem ?EPOCH_SIZE. + +%% Reconstruct session counter by adding most significant bits from +%% the current counter to the packet id. +-spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno(). +packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) -> + packet_id_to_seqno_(NextSeqno, PacketId). + %%================================================================================ %% Internal exports %%================================================================================ @@ -150,18 +209,34 @@ poll(SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < 16#7fff %%================================================================================ compute_inflight_range([]) -> - {1, 1}; + {#{ack => 1, comp => 1}, 1}; compute_inflight_range(Ranges) -> _RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges), - RangesUnacked = lists:dropwhile( - fun(#ds_pubrange{type = T}) -> T == checkpoint end, + AckedUntil = find_committed_until(ack, Ranges), + CompUntil = find_committed_until(comp, Ranges), + Commits = #{ + ack => emqx_maybe:define(AckedUntil, LastSeqno), + comp => emqx_maybe:define(CompUntil, LastSeqno) + }, + {Commits, LastSeqno}. + +find_committed_until(Track, Ranges) -> + RangesUncommitted = lists:dropwhile( + fun(Range) -> + case Range of + #ds_pubrange{type = ?T_CHECKPOINT} -> + true; + #ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} -> + not has_track(Track, Tracks) + end + end, Ranges ), - case RangesUnacked of - [#ds_pubrange{id = {_, AckedUntil}} | _] -> - {AckedUntil, LastSeqno}; + case RangesUncommitted of + [#ds_pubrange{id = {_, CommittedUntil}} | _] -> + CommittedUntil; [] -> - {LastSeqno, LastSeqno} + undefined end. -spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()]. @@ -173,21 +248,22 @@ get_ranges(SessionId) -> ), mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read). -fetch(SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 -> +fetch(ReplyFun, SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 -> #inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0, ItBegin = get_last_iterator(DSStream, Ranges), {ok, ItEnd, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N), case Messages of [] -> - fetch(SessionId, Inflight0, Streams, N, Acc); + fetch(ReplyFun, SessionId, Inflight0, Streams, N, Acc); _ -> - {Publishes, UntilSeqno} = publish(FirstSeqno, Messages, _PreserveQoS0 = true), - Size = range_size(FirstSeqno, UntilSeqno), %% We need to preserve the iterator pointing to the beginning of the %% range, so that we can replay it if needed. + {Publishes, {UntilSeqno, Tracks}} = publish(ReplyFun, FirstSeqno, Messages), + Size = range_size(FirstSeqno, UntilSeqno), Range0 = #ds_pubrange{ id = {SessionId, FirstSeqno}, - type = inflight, + type = ?T_INFLIGHT, + tracks = Tracks, until = UntilSeqno, stream = DSStream#ds_stream.ref, iterator = ItBegin @@ -196,25 +272,25 @@ fetch(SessionId, Inflight0, [DSStream | Streams], N, Acc) when N > 0 -> %% ...Yet we need to keep the iterator pointing past the end of the %% range, so that we can pick up where we left off: it will become %% `ItBegin` of the next range for this stream. - Range = Range0#ds_pubrange{iterator = ItEnd}, + Range = keep_next_iterator(ItEnd, Range0), Inflight = Inflight0#inflight{ next_seqno = UntilSeqno, offset_ranges = Ranges ++ [Range] }, - fetch(SessionId, Inflight, Streams, N - Size, [Publishes | Acc]) + fetch(ReplyFun, SessionId, Inflight, Streams, N - Size, [Publishes | Acc]) end; -fetch(_SessionId, Inflight, _Streams, _N, Acc) -> +fetch(_ReplyFun, _SessionId, Inflight, _Streams, _N, Acc) -> Publishes = lists:append(lists:reverse(Acc)), {Publishes, Inflight}. -discard_acked( +discard_committed( SessionId, - Inflight0 = #inflight{acked_until = AckedUntil, offset_ranges = Ranges0} + Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0} ) -> %% TODO: This could be kept and incrementally updated in the inflight state. Checkpoints = find_checkpoints(Ranges0), %% TODO: Wrap this in `mria:async_dirty/2`? - Ranges = discard_acked_ranges(SessionId, AckedUntil, Checkpoints, Ranges0), + Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0), Inflight0#inflight{offset_ranges = Ranges}. find_checkpoints(Ranges) -> @@ -227,84 +303,178 @@ find_checkpoints(Ranges) -> Ranges ). -discard_acked_ranges( +discard_committed_ranges( SessionId, - AckedUntil, + Commits, Checkpoints, - [Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest] -) when Until =< AckedUntil -> - %% This range has been fully acked. - %% Either discard it completely, or preserve the iterator for the next range - %% over this stream (i.e. a checkpoint). - RangeKept = - case maps:get(StreamRef, Checkpoints) of - CP when CP > Until -> - discard_range(Range), - []; - Until -> - [checkpoint_range(Range)] + Ranges = [Range = #ds_pubrange{until = Until, stream = StreamRef} | Rest] +) -> + case discard_committed_range(Commits, Range) of + discard -> + %% This range has been fully committed. + %% Either discard it completely, or preserve the iterator for the next range + %% over this stream (i.e. a checkpoint). + RangeKept = + case maps:get(StreamRef, Checkpoints) of + CP when CP > Until -> + discard_range(Range), + []; + Until -> + [checkpoint_range(Range)] + end, + %% Since we're (intentionally) not using transactions here, it's important to + %% issue database writes in the same order in which ranges are stored: from + %% the oldest to the newest. This is also why we need to compute which ranges + %% should become checkpoints before we start writing anything. + RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest); + keep -> + %% This range has not been fully committed. + [Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]; + keep_all -> + %% The rest of ranges (if any) still have uncommitted messages. + Ranges; + TracksLeft -> + %% Only some track has been committed. + %% Preserve the uncommitted tracks in the database. + RangeKept = Range#ds_pubrange{tracks = TracksLeft}, + preserve_range(restore_first_iterator(RangeKept)), + [RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)] + end; +discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) -> + []. + +discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) -> + discard; +discard_committed_range( + #{ack := AckedUntil, comp := CompUntil}, + #ds_pubrange{until = Until} +) when Until > AckedUntil andalso Until > CompUntil -> + keep_all; +discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) -> + case discard_tracks(Commits, Until, Tracks) of + 0 -> + discard; + Tracks -> + keep; + TracksLeft -> + TracksLeft + end. + +discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) -> + TAck = + case Until > AckedUntil of + true -> ?TRACK_FLAG(?ACK) band Tracks; + false -> 0 end, - %% Since we're (intentionally) not using transactions here, it's important to - %% issue database writes in the same order in which ranges are stored: from - %% the oldest to the newest. This is also why we need to compute which ranges - %% should become checkpoints before we start writing anything. - RangeKept ++ discard_acked_ranges(SessionId, AckedUntil, Checkpoints, Rest); -discard_acked_ranges(_SessionId, _AckedUntil, _Checkpoints, Ranges) -> - %% The rest of ranges (if any) still have unacked messages. - Ranges. + TComp = + case Until > CompUntil of + true -> ?TRACK_FLAG(?COMP) band Tracks; + false -> 0 + end, + TAck bor TComp. replay_range( - Range0 = #ds_pubrange{type = inflight, id = {_, First}, until = Until, iterator = It}, - AckedUntil, + ReplyFun, + Range0 = #ds_pubrange{type = ?T_INFLIGHT, id = {_, First}, until = Until, iterator = It}, Acc ) -> Size = range_size(First, Until), - FirstUnacked = max(First, AckedUntil), - {ok, ItNext, Messages} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size), - MessagesUnacked = - case FirstUnacked of - First -> - Messages; - _ -> - lists:nthtail(range_size(First, FirstUnacked), Messages) - end, - MessagesReplay = [emqx_message:set_flag(dup, true, Msg) || Msg <- MessagesUnacked], + {ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size), %% Asserting that range is consistent with the message storage state. - {Replies, Until} = publish(FirstUnacked, MessagesReplay, _PreserveQoS0 = false), + {Replies, {Until, _TracksInitial}} = publish(ReplyFun, First, MessagesUnacked), %% Again, we need to keep the iterator pointing past the end of the %% range, so that we can pick up where we left off. - Range = Range0#ds_pubrange{iterator = ItNext}, + Range = keep_next_iterator(ItNext, Range0), {Range, Replies ++ Acc}; -replay_range(Range0 = #ds_pubrange{type = checkpoint}, _AckedUntil, Acc) -> +replay_range(_ReplyFun, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) -> {Range0, Acc}. -publish(FirstSeqNo, Messages, PreserveQos0) -> - do_publish(FirstSeqNo, Messages, PreserveQos0, []). +validate_commit( + Track, + PacketId, + Inflight = #inflight{commits = Commits, next_seqno = NextSeqno} +) -> + Seqno = packet_id_to_seqno_(NextSeqno, PacketId), + CommittedUntil = maps:get(Track, Commits), + CommitNext = get_commit_next(Track, Inflight), + case Seqno >= CommittedUntil andalso Seqno < CommitNext of + true -> + next_seqno(Seqno); + false -> + ?SLOG(warning, #{ + msg => "out-of-order_commit", + track => Track, + packet_id => PacketId, + commit_seqno => Seqno, + committed_until => CommittedUntil, + commit_next => CommitNext + }), + false + end. -do_publish(SeqNo, [], _, Acc) -> - {lists:reverse(Acc), SeqNo}; -do_publish(SeqNo, [#message{qos = 0} | Messages], false, Acc) -> - do_publish(SeqNo, Messages, false, Acc); -do_publish(SeqNo, [#message{qos = 0} = Message | Messages], true, Acc) -> - do_publish(SeqNo, Messages, true, [{undefined, Message} | Acc]); -do_publish(SeqNo, [Message | Messages], PreserveQos0, Acc) -> - PacketId = seqno_to_packet_id(SeqNo), - do_publish(next_seqno(SeqNo), Messages, PreserveQos0, [{PacketId, Message} | Acc]). +get_commit_next(ack, #inflight{next_seqno = NextSeqno}) -> + NextSeqno; +get_commit_next(rec, #inflight{next_seqno = NextSeqno}) -> + NextSeqno; +get_commit_next(comp, #inflight{commits = Commits}) -> + maps:get(rec, Commits). + +publish(ReplyFun, FirstSeqno, Messages) -> + lists:mapfoldl( + fun(Message, {Seqno, TAcc}) -> + case ReplyFun(Seqno, Message) of + {_Advance = false, Reply} -> + {Reply, {Seqno, TAcc}}; + Reply -> + NextSeqno = next_seqno(Seqno), + NextTAcc = add_msg_track(Message, TAcc), + {Reply, {NextSeqno, NextTAcc}} + end + end, + {FirstSeqno, 0}, + Messages + ). + +add_msg_track(Message, Tracks) -> + case emqx_message:qos(Message) of + 1 -> ?TRACK_FLAG(?ACK) bor Tracks; + 2 -> ?TRACK_FLAG(?COMP) bor Tracks; + _ -> Tracks + end. + +keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) -> + Range#ds_pubrange{ + iterator = ItNext, + %% We need to keep the first iterator around, in case we need to preserve + %% this range again, updating still uncommitted tracks it's part of. + misc = Misc#{iterator_first => ItFirst} + }. + +restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) -> + Range#ds_pubrange{ + iterator = ItFirst, + misc = maps:remove(iterator_first, Misc) + }. -spec preserve_range(ds_pubrange()) -> ok. -preserve_range(Range = #ds_pubrange{type = inflight}) -> +preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) -> mria:dirty_write(?SESSION_PUBRANGE_TAB, Range). +has_track(ack, Tracks) -> + (?TRACK_FLAG(?ACK) band Tracks) > 0; +has_track(comp, Tracks) -> + (?TRACK_FLAG(?COMP) band Tracks) > 0. + -spec discard_range(ds_pubrange()) -> ok. discard_range(#ds_pubrange{id = RangeId}) -> mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId). -spec checkpoint_range(ds_pubrange()) -> ds_pubrange(). -checkpoint_range(Range0 = #ds_pubrange{type = inflight}) -> - Range = Range0#ds_pubrange{type = checkpoint}, +checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) -> + Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}}, ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range), Range; -checkpoint_range(Range = #ds_pubrange{type = checkpoint}) -> +checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) -> %% This range should have been checkpointed already. Range. @@ -320,6 +490,21 @@ get_last_iterator(DSStream = #ds_stream{ref = StreamRef}, Ranges) -> get_streams(SessionId) -> mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId). +-spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno(). +get_committed_offset(SessionId, Name) -> + case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of + [] -> + 1; + [#ds_committed_offset{until = Seqno}] -> + Seqno + end. + +-spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok. +update_committed_offset(SessionId, Name, Until) -> + mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{ + id = {SessionId, Name}, until = Until + }). + next_seqno(Seqno) -> NextSeqno = Seqno + 1, case seqno_to_packet_id(NextSeqno) of @@ -332,26 +517,15 @@ next_seqno(Seqno) -> NextSeqno end. -%% Reconstruct session counter by adding most significant bits from -%% the current counter to the packet id. --spec packet_id_to_seqno(_Next :: seqno(), emqx_types:packet_id()) -> seqno(). -packet_id_to_seqno(NextSeqNo, PacketId) -> - Epoch = NextSeqNo bsr 16, - case packet_id_to_seqno_(Epoch, PacketId) of - N when N =< NextSeqNo -> +packet_id_to_seqno_(NextSeqno, PacketId) -> + Epoch = NextSeqno bsr 16, + case (Epoch bsl 16) + PacketId of + N when N =< NextSeqno -> N; - _ -> - packet_id_to_seqno_(Epoch - 1, PacketId) + N -> + N - ?EPOCH_SIZE end. --spec packet_id_to_seqno_(non_neg_integer(), emqx_types:packet_id()) -> seqno(). -packet_id_to_seqno_(Epoch, PacketId) -> - (Epoch bsl 16) + PacketId. - --spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0. -seqno_to_packet_id(Seqno) -> - Seqno rem 16#10000. - range_size(FirstSeqno, UntilSeqno) -> %% This function assumes that gaps in the sequence ID occur _only_ when the %% packet ID wraps. @@ -379,19 +553,19 @@ ro_transaction(Fun) -> %% This test only tests boundary conditions (to make sure property-based test didn't skip them): packet_id_to_seqno_test() -> %% Packet ID = 1; first epoch: - ?assertEqual(1, packet_id_to_seqno(1, 1)), - ?assertEqual(1, packet_id_to_seqno(10, 1)), - ?assertEqual(1, packet_id_to_seqno(1 bsl 16 - 1, 1)), - ?assertEqual(1, packet_id_to_seqno(1 bsl 16, 1)), + ?assertEqual(1, packet_id_to_seqno_(1, 1)), + ?assertEqual(1, packet_id_to_seqno_(10, 1)), + ?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)), + ?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)), %% Packet ID = 1; second and 3rd epochs: - ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(1 bsl 16 + 1, 1)), - ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno(2 bsl 16, 1)), - ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno(2 bsl 16 + 1, 1)), + ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)), + ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)), + ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)), %% Packet ID = 16#ffff: PID = 1 bsl 16 - 1, - ?assertEqual(PID, packet_id_to_seqno(PID, PID)), - ?assertEqual(PID, packet_id_to_seqno(1 bsl 16, PID)), - ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno(2 bsl 16, PID)), + ?assertEqual(PID, packet_id_to_seqno_(PID, PID)), + ?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)), + ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)), ok. packet_id_to_seqno_test_() -> @@ -406,8 +580,8 @@ packet_id_to_seqno_prop() -> SeqNo, seqno_gen(NextSeqNo), begin - PacketId = SeqNo rem 16#10000, - ?assertEqual(SeqNo, packet_id_to_seqno(NextSeqNo, PacketId)), + PacketId = seqno_to_packet_id(SeqNo), + ?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)), true end ) @@ -437,27 +611,42 @@ range_size_test_() -> compute_inflight_range_test_() -> [ ?_assertEqual( - {1, 1}, + {#{ack => 1, comp => 1}, 1}, compute_inflight_range([]) ), ?_assertEqual( - {12, 42}, + {#{ack => 12, comp => 13}, 42}, compute_inflight_range([ - #ds_pubrange{id = {<<>>, 1}, until = 2, type = checkpoint}, - #ds_pubrange{id = {<<>>, 4}, until = 8, type = checkpoint}, - #ds_pubrange{id = {<<>>, 11}, until = 12, type = checkpoint}, - #ds_pubrange{id = {<<>>, 12}, until = 13, type = inflight}, - #ds_pubrange{id = {<<>>, 13}, until = 20, type = inflight}, - #ds_pubrange{id = {<<>>, 20}, until = 42, type = inflight} + #ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT}, + #ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT}, + #ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT}, + #ds_pubrange{ + id = {<<>>, 12}, + until = 13, + type = ?T_INFLIGHT, + tracks = ?TRACK_FLAG(?ACK) + }, + #ds_pubrange{ + id = {<<>>, 13}, + until = 20, + type = ?T_INFLIGHT, + tracks = ?TRACK_FLAG(?COMP) + }, + #ds_pubrange{ + id = {<<>>, 20}, + until = 42, + type = ?T_INFLIGHT, + tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP) + } ]) ), ?_assertEqual( - {13, 13}, + {#{ack => 13, comp => 13}, 13}, compute_inflight_range([ - #ds_pubrange{id = {<<>>, 1}, until = 2, type = checkpoint}, - #ds_pubrange{id = {<<>>, 4}, until = 8, type = checkpoint}, - #ds_pubrange{id = {<<>>, 11}, until = 12, type = checkpoint}, - #ds_pubrange{id = {<<>>, 12}, until = 13, type = checkpoint} + #ds_pubrange{id = {<<>>, 1}, until = 2, type = ?T_CHECKPOINT}, + #ds_pubrange{id = {<<>>, 4}, until = 8, type = ?T_CHECKPOINT}, + #ds_pubrange{id = {<<>>, 11}, until = 12, type = ?T_CHECKPOINT}, + #ds_pubrange{id = {<<>>, 12}, until = 13, type = ?T_CHECKPOINT} ]) ) ]. diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 76b54e34a..9844e6d48 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -56,13 +56,16 @@ deliver/3, replay/3, handle_timeout/3, - disconnect/1, + disconnect/2, terminate/2 ]). %% session table operations -export([create_tables/0]). +%% internal export used by session GC process +-export([destroy_session/1]). + %% Remove me later (satisfy checks for an unused BPAPI) -export([ do_open_iterator/3, @@ -74,7 +77,7 @@ -ifdef(TEST). -export([ - session_open/1, + session_open/2, list_all_sessions/0, list_all_subscriptions/0, list_all_streams/0, @@ -98,22 +101,26 @@ id := id(), %% When the session was created created_at := timestamp(), - %% When the session should expire - expires_at := timestamp() | never, + %% When the client was last considered alive + last_alive_at := timestamp(), %% Client’s Subscriptions. subscriptions := #{topic_filter() => subscription()}, %% Inflight messages inflight := emqx_persistent_message_ds_replayer:inflight(), %% Receive maximum receive_maximum := pos_integer(), + %% Connection Info + conninfo := emqx_types:conninfo(), %% props := map() }. -type timestamp() :: emqx_utils_calendar:epoch_millisecond(). +-type millisecond() :: non_neg_integer(). -type clientinfo() :: emqx_types:clientinfo(). -type conninfo() :: emqx_session:conninfo(). -type replies() :: emqx_session:replies(). +-type timer() :: pull | get_streams | bump_last_alive_at. -define(STATS_KEYS, [ subscriptions_cnt, @@ -123,6 +130,12 @@ next_pkt_id ]). +-define(IS_EXPIRED(NOW_MS, LAST_ALIVE_AT, EI), + (is_number(LAST_ALIVE_AT) andalso + is_number(EI) andalso + (NOW_MS >= LAST_ALIVE_AT + EI)) +). + -export_type([id/0]). %% @@ -144,26 +157,24 @@ open(#{clientid := ClientID} = _ClientInfo, ConnInfo) -> %% somehow isolate those idling not-yet-expired sessions into a separate process %% space, and move this call back into `emqx_cm` where it belongs. ok = emqx_cm:discard_session(ClientID), - case maps:get(clean_start, ConnInfo, false) of + case session_open(ClientID, ConnInfo) of + Session0 = #{} -> + ensure_timers(), + ReceiveMaximum = receive_maximum(ConnInfo), + Session = Session0#{receive_maximum => ReceiveMaximum}, + {true, Session, []}; false -> - case session_open(ClientID) of - Session0 = #{} -> - ensure_timers(), - ReceiveMaximum = receive_maximum(ConnInfo), - Session = Session0#{receive_maximum => ReceiveMaximum}, - {true, Session, []}; - false -> - false - end; - true -> - session_drop(ClientID), false end. ensure_session(ClientID, ConnInfo, Conf) -> - Session = session_ensure_new(ClientID, Conf), + Session = session_ensure_new(ClientID, ConnInfo, Conf), ReceiveMaximum = receive_maximum(ConnInfo), - Session#{subscriptions => #{}, receive_maximum => ReceiveMaximum}. + Session#{ + conninfo => ConnInfo, + receive_maximum => ReceiveMaximum, + subscriptions => #{} + }. -spec destroy(session() | clientinfo()) -> ok. destroy(#{id := ClientID}) -> @@ -239,6 +250,7 @@ print_session(ClientId) -> session => Session, streams => mnesia:read(?SESSION_STREAM_TAB, ClientId), pubranges => session_read_pubranges(ClientId), + offsets => session_read_offsets(ClientId), subscriptions => session_read_subscriptions(ClientId) }; [] -> @@ -319,12 +331,13 @@ publish(_PacketId, Msg, Session) -> {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}. puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) -> - case emqx_persistent_message_ds_replayer:commit_offset(Id, PacketId, Inflight0) of + case emqx_persistent_message_ds_replayer:commit_offset(Id, ack, PacketId, Inflight0) of {true, Inflight} -> %% TODO - Msg = #message{}, + Msg = emqx_message:make(Id, <<>>, <<>>), {ok, Msg, [], Session#{inflight => Inflight}}; {false, _} -> + %% Invalid Packet Id {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} end. @@ -335,9 +348,16 @@ puback(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) -> -spec pubrec(emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), session()} | {error, emqx_types:reason_code()}. -pubrec(_PacketId, _Session = #{}) -> - % TODO: stub - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. +pubrec(PacketId, Session = #{id := Id, inflight := Inflight0}) -> + case emqx_persistent_message_ds_replayer:commit_offset(Id, rec, PacketId, Inflight0) of + {true, Inflight} -> + %% TODO + Msg = emqx_message:make(Id, <<>>, <<>>), + {ok, Msg, Session#{inflight => Inflight}}; + {false, _} -> + %% Invalid Packet Id + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. %%-------------------------------------------------------------------- %% Client -> Broker: PUBREL @@ -356,9 +376,16 @@ pubrel(_PacketId, Session = #{}) -> -spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) -> {ok, emqx_types:message(), replies(), session()} | {error, emqx_types:reason_code()}. -pubcomp(_ClientInfo, _PacketId, _Session = #{}) -> - % TODO: stub - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. +pubcomp(_ClientInfo, PacketId, Session = #{id := Id, inflight := Inflight0}) -> + case emqx_persistent_message_ds_replayer:commit_offset(Id, comp, PacketId, Inflight0) of + {true, Inflight} -> + %% TODO + Msg = emqx_message:make(Id, <<>>, <<>>), + {ok, Msg, [], Session#{inflight => Inflight}}; + {false, _} -> + %% Invalid Packet Id + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. %%-------------------------------------------------------------------- @@ -375,7 +402,18 @@ handle_timeout( pull, Session = #{id := Id, inflight := Inflight0, receive_maximum := ReceiveMaximum} ) -> - {Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll(Id, Inflight0, ReceiveMaximum), + {Publishes, Inflight} = emqx_persistent_message_ds_replayer:poll( + fun + (_Seqno, Message = #message{qos = ?QOS_0}) -> + {false, {undefined, Message}}; + (Seqno, Message) -> + PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno), + {PacketId, Message} + end, + Id, + Inflight0, + ReceiveMaximum + ), IdlePollInterval = emqx_config:get([session_persistence, idle_poll_interval]), Timeout = case Publishes of @@ -385,22 +423,50 @@ handle_timeout( 0 end, ensure_timer(pull, Timeout), - {ok, Publishes, Session#{inflight => Inflight}}; + {ok, Publishes, Session#{inflight := Inflight}}; handle_timeout(_ClientInfo, get_streams, Session) -> renew_streams(Session), ensure_timer(get_streams), + {ok, [], Session}; +handle_timeout(_ClientInfo, bump_last_alive_at, Session0) -> + %% Note: we take a pessimistic approach here and assume that the client will be alive + %% until the next bump timeout. With this, we avoid garbage collecting this session + %% too early in case the session/connection/node crashes earlier without having time + %% to commit the time. + BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]), + EstimatedLastAliveAt = now_ms() + BumpInterval, + Session = session_set_last_alive_at_trans(Session0, EstimatedLastAliveAt), + ensure_timer(bump_last_alive_at), {ok, [], Session}. -spec replay(clientinfo(), [], session()) -> {ok, replies(), session()}. replay(_ClientInfo, [], Session = #{inflight := Inflight0}) -> - {Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(Inflight0), + AckedUntil = emqx_persistent_message_ds_replayer:committed_until(ack, Inflight0), + RecUntil = emqx_persistent_message_ds_replayer:committed_until(rec, Inflight0), + CompUntil = emqx_persistent_message_ds_replayer:committed_until(comp, Inflight0), + ReplyFun = fun + (_Seqno, #message{qos = ?QOS_0}) -> + {false, []}; + (Seqno, #message{qos = ?QOS_1}) when Seqno < AckedUntil -> + []; + (Seqno, #message{qos = ?QOS_2}) when Seqno < CompUntil -> + []; + (Seqno, #message{qos = ?QOS_2}) when Seqno < RecUntil -> + PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno), + {pubrel, PacketId}; + (Seqno, Message) -> + PacketId = emqx_persistent_message_ds_replayer:seqno_to_packet_id(Seqno), + {PacketId, emqx_message:set_flag(dup, true, Message)} + end, + {Replies, Inflight} = emqx_persistent_message_ds_replayer:replay(ReplyFun, Inflight0), {ok, Replies, Session#{inflight := Inflight}}. %%-------------------------------------------------------------------- --spec disconnect(session()) -> {shutdown, session()}. -disconnect(Session = #{}) -> +-spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}. +disconnect(Session0, ConnInfo) -> + Session = session_set_last_alive_at_trans(Session0, ConnInfo, now_ms()), {shutdown, Session}. -spec terminate(Reason :: term(), session()) -> ok. @@ -507,11 +573,22 @@ create_tables() -> {attributes, record_info(fields, ds_pubrange)} ] ), + ok = mria:create_table( + ?SESSION_COMMITTED_OFFSET_TAB, + [ + {rlog_shard, ?DS_MRIA_SHARD}, + {type, set}, + {storage, storage()}, + {record_name, ds_committed_offset}, + {attributes, record_info(fields, ds_committed_offset)} + ] + ), ok = mria:wait_for_tables([ ?SESSION_TAB, ?SESSION_SUBSCRIPTIONS_TAB, ?SESSION_STREAM_TAB, - ?SESSION_PUBRANGE_TAB + ?SESSION_PUBRANGE_TAB, + ?SESSION_COMMITTED_OFFSET_TAB ]), ok. @@ -530,47 +607,84 @@ storage() -> %% %% Note: session API doesn't handle session takeovers, it's the job of %% the broker. --spec session_open(id()) -> +-spec session_open(id(), emqx_types:conninfo()) -> session() | false. -session_open(SessionId) -> - ro_transaction(fun() -> +session_open(SessionId, NewConnInfo) -> + NowMS = now_ms(), + transaction(fun() -> case mnesia:read(?SESSION_TAB, SessionId, write) of - [Record = #session{}] -> - Session = export_session(Record), - DSSubs = session_read_subscriptions(SessionId), - Subscriptions = export_subscriptions(DSSubs), - Inflight = emqx_persistent_message_ds_replayer:open(SessionId), - Session#{ - subscriptions => Subscriptions, - inflight => Inflight - }; - [] -> + [Record0 = #session{last_alive_at = LastAliveAt, conninfo = ConnInfo}] -> + EI = expiry_interval(ConnInfo), + case ?IS_EXPIRED(NowMS, LastAliveAt, EI) of + true -> + session_drop(SessionId), + false; + false -> + %% new connection being established + Record1 = Record0#session{conninfo = NewConnInfo}, + Record = session_set_last_alive_at(Record1, NowMS), + Session = export_session(Record), + DSSubs = session_read_subscriptions(SessionId), + Subscriptions = export_subscriptions(DSSubs), + Inflight = emqx_persistent_message_ds_replayer:open(SessionId), + Session#{ + conninfo => NewConnInfo, + inflight => Inflight, + subscriptions => Subscriptions + } + end; + _ -> false end end). --spec session_ensure_new(id(), _Props :: map()) -> +-spec session_ensure_new(id(), emqx_types:conninfo(), _Props :: map()) -> session(). -session_ensure_new(SessionId, Props) -> +session_ensure_new(SessionId, ConnInfo, Props) -> transaction(fun() -> ok = session_drop_subscriptions(SessionId), - Session = export_session(session_create(SessionId, Props)), + Session = export_session(session_create(SessionId, ConnInfo, Props)), Session#{ subscriptions => #{}, inflight => emqx_persistent_message_ds_replayer:new() } end). -session_create(SessionId, Props) -> +session_create(SessionId, ConnInfo, Props) -> Session = #session{ id = SessionId, - created_at = erlang:system_time(millisecond), - expires_at = never, + created_at = now_ms(), + last_alive_at = now_ms(), + conninfo = ConnInfo, props = Props }, ok = mnesia:write(?SESSION_TAB, Session, write), Session. +session_set_last_alive_at_trans(Session, LastAliveAt) -> + #{conninfo := ConnInfo} = Session, + session_set_last_alive_at_trans(Session, ConnInfo, LastAliveAt). + +session_set_last_alive_at_trans(Session, NewConnInfo, LastAliveAt) -> + #{id := SessionId} = Session, + transaction(fun() -> + case mnesia:read(?SESSION_TAB, SessionId, write) of + [#session{} = SessionRecord0] -> + SessionRecord = SessionRecord0#session{conninfo = NewConnInfo}, + _ = session_set_last_alive_at(SessionRecord, LastAliveAt), + ok; + _ -> + %% log and crash? + ok + end + end), + Session#{conninfo := NewConnInfo, last_alive_at := LastAliveAt}. + +session_set_last_alive_at(SessionRecord0, LastAliveAt) -> + SessionRecord = SessionRecord0#session{last_alive_at = LastAliveAt}, + ok = mnesia:write(?SESSION_TAB, SessionRecord, write), + SessionRecord. + %% @doc Called when a client reconnects with `clean session=true' or %% during session GC -spec session_drop(id()) -> ok. @@ -578,6 +692,7 @@ session_drop(DSSessionId) -> transaction(fun() -> ok = session_drop_subscriptions(DSSessionId), ok = session_drop_pubranges(DSSessionId), + ok = session_drop_offsets(DSSessionId), ok = session_drop_streams(DSSessionId), ok = mnesia:delete(?SESSION_TAB, DSSessionId, write) end). @@ -669,11 +784,22 @@ session_read_pubranges(DSSessionId, LockKind) -> ), mnesia:select(?SESSION_PUBRANGE_TAB, MS, LockKind). +session_read_offsets(DSSessionID) -> + session_read_offsets(DSSessionID, read). + +session_read_offsets(DSSessionId, LockKind) -> + MS = ets:fun2ms( + fun(#ds_committed_offset{id = {Sess, Type}}) when Sess =:= DSSessionId -> + {DSSessionId, Type} + end + ), + mnesia:select(?SESSION_COMMITTED_OFFSET_TAB, MS, LockKind). + -spec new_subscription_id(id(), topic_filter()) -> {subscription_id(), integer()}. new_subscription_id(DSSessionId, TopicFilter) -> %% Note: here we use _milliseconds_ to match with the timestamp %% field of `#message' record. - NowMS = erlang:system_time(millisecond), + NowMS = now_ms(), DSSubId = {DSSessionId, TopicFilter}, {DSSubId, NowMS}. @@ -681,6 +807,9 @@ new_subscription_id(DSSessionId, TopicFilter) -> subscription_id_to_topic_filter({_DSSessionId, TopicFilter}) -> TopicFilter. +now_ms() -> + erlang:system_time(millisecond). + %%-------------------------------------------------------------------- %% RPC targets (v1) %%-------------------------------------------------------------------- @@ -778,11 +907,27 @@ session_drop_pubranges(DSSessionId) -> RangeIds ). +%% must be called inside a transaction +-spec session_drop_offsets(id()) -> ok. +session_drop_offsets(DSSessionId) -> + OffsetIds = session_read_offsets(DSSessionId, write), + lists:foreach( + fun(OffsetId) -> + mnesia:delete(?SESSION_COMMITTED_OFFSET_TAB, OffsetId, write) + end, + OffsetIds + ). + %%-------------------------------------------------------------------------------- transaction(Fun) -> - {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun), - Res. + case mnesia:is_transaction() of + true -> + Fun(); + false -> + {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun), + Res + end. ro_transaction(Fun) -> {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun), @@ -800,7 +945,7 @@ export_subscriptions(DSSubs) -> ). export_session(#session{} = Record) -> - export_record(Record, #session.id, [id, created_at, expires_at, props], #{}). + export_record(Record, #session.id, [id, created_at, last_alive_at, conninfo, props], #{}). export_subscription(#ds_sub{} = Record) -> export_record(Record, #ds_sub.start_time, [start_time, props, extra], #{}). @@ -814,13 +959,17 @@ export_record(_, _, [], Acc) -> %% effects. Add `CBM:init' callback to the session behavior? ensure_timers() -> ensure_timer(pull), - ensure_timer(get_streams). + ensure_timer(get_streams), + ensure_timer(bump_last_alive_at). --spec ensure_timer(pull | get_streams) -> ok. +-spec ensure_timer(timer()) -> ok. +ensure_timer(bump_last_alive_at = Type) -> + BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]), + ensure_timer(Type, BumpInterval); ensure_timer(Type) -> ensure_timer(Type, 100). --spec ensure_timer(pull | get_streams, non_neg_integer()) -> ok. +-spec ensure_timer(timer(), non_neg_integer()) -> ok. ensure_timer(Type, Timeout) -> _ = emqx_utils:start_timer(Timeout, {emqx_session, Type}), ok. @@ -832,11 +981,24 @@ receive_maximum(ConnInfo) -> %% indicates that it's optional. maps:get(receive_maximum, ConnInfo, 65_535). +-spec expiry_interval(conninfo()) -> millisecond(). +expiry_interval(ConnInfo) -> + maps:get(expiry_interval, ConnInfo, 0). + -ifdef(TEST). list_all_sessions() -> DSSessionIds = mnesia:dirty_all_keys(?SESSION_TAB), - Sessions = lists:map( - fun(SessionID) -> {SessionID, session_open(SessionID)} end, + ConnInfo = #{}, + Sessions = lists:filtermap( + fun(SessionID) -> + Sess = session_open(SessionID, ConnInfo), + case Sess of + false -> + false; + _ -> + {true, {SessionID, Sess}} + end + end, DSSessionIds ), maps:from_list(Sessions). diff --git a/apps/emqx/src/emqx_persistent_session_ds.hrl b/apps/emqx/src/emqx_persistent_session_ds.hrl index 653ac444a..306e63b2e 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds.hrl @@ -22,8 +22,12 @@ -define(SESSION_SUBSCRIPTIONS_TAB, emqx_ds_session_subscriptions). -define(SESSION_STREAM_TAB, emqx_ds_stream_tab). -define(SESSION_PUBRANGE_TAB, emqx_ds_pubrange_tab). +-define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab). -define(DS_MRIA_SHARD, emqx_ds_session_shard). +-define(T_INFLIGHT, 1). +-define(T_CHECKPOINT, 2). + -record(ds_sub, { id :: emqx_persistent_session_ds:subscription_id(), start_time :: emqx_ds:time(), @@ -56,7 +60,11 @@ %% * Inflight range is a range of yet unacked messages from this stream. %% * Checkpoint range was already acked, its purpose is to keep track of the %% very last iterator for this stream. - type :: inflight | checkpoint, + type :: ?T_INFLIGHT | ?T_CHECKPOINT, + %% What commit tracks this range is part of. + %% This is rarely stored: we only need to persist it when the range contains + %% QoS 2 messages. + tracks = 0 :: non_neg_integer(), %% Meaning of this depends on the type of the range: %% * For inflight range, this is the iterator pointing to the first message in %% the range. @@ -68,12 +76,24 @@ }). -type ds_pubrange() :: #ds_pubrange{}. +-record(ds_committed_offset, { + id :: { + %% What session this marker belongs to. + _Session :: emqx_persistent_session_ds:id(), + %% Marker name. + _CommitType + }, + %% Where this marker is pointing to: the first seqno that is not marked. + until :: emqx_persistent_message_ds_replayer:seqno() +}). + -record(session, { %% same as clientid id :: emqx_persistent_session_ds:id(), %% creation time created_at :: _Millisecond :: non_neg_integer(), - expires_at = never :: _Millisecond :: non_neg_integer() | never, + last_alive_at :: _Millisecond :: non_neg_integer(), + conninfo :: emqx_types:conninfo(), %% for future usage props = #{} :: map() }). diff --git a/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl b/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl new file mode 100644 index 000000000..bf607804f --- /dev/null +++ b/apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl @@ -0,0 +1,161 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_persistent_session_ds_gc_worker). + +-behaviour(gen_server). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("stdlib/include/qlc.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +-include("emqx_persistent_session_ds.hrl"). + +%% API +-export([ + start_link/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%% call/cast/info records +-record(gc, {}). + +%%-------------------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%-------------------------------------------------------------------------------- +%% `gen_server' API +%%-------------------------------------------------------------------------------- + +init(_Opts) -> + ensure_gc_timer(), + State = #{}, + {ok, State}. + +handle_call(_Call, _From, State) -> + {reply, error, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info(#gc{}, State) -> + try_gc(), + ensure_gc_timer(), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------------------- +%% Internal fns +%%-------------------------------------------------------------------------------- + +ensure_gc_timer() -> + Timeout = emqx_config:get([session_persistence, session_gc_interval]), + _ = erlang:send_after(Timeout, self(), #gc{}), + ok. + +try_gc() -> + %% Only cores should run GC. + CoreNodes = mria_membership:running_core_nodelist(), + Res = global:trans( + {?MODULE, self()}, + fun() -> ?tp_span(ds_session_gc, #{}, start_gc()) end, + CoreNodes, + %% Note: we set retries to 1 here because, in rare occasions, GC might start at the + %% same time in more than one node, and each one will abort the other. By allowing + %% one retry, at least one node will (hopefully) get to enter the transaction and + %% the other will abort. If GC runs too fast, both nodes might run in sequence. + %% But, in that case, GC is clearly not too costly, and that shouldn't be a problem, + %% resource-wise. + _Retries = 1 + ), + case Res of + aborted -> + ?tp(ds_session_gc_lock_taken, #{}), + ok; + ok -> + ok + end. + +now_ms() -> + erlang:system_time(millisecond). + +start_gc() -> + do_gc(more). + +zombie_session_ms() -> + NowMS = now_ms(), + GCInterval = emqx_config:get([session_persistence, session_gc_interval]), + BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]), + TimeThreshold = max(GCInterval, BumpInterval) * 3, + ets:fun2ms( + fun( + #session{ + id = DSSessionId, + last_alive_at = LastAliveAt, + conninfo = #{expiry_interval := EI} + } + ) when + LastAliveAt + EI + TimeThreshold =< NowMS + -> + DSSessionId + end + ). + +do_gc(more) -> + GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]), + MS = zombie_session_ms(), + {atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() -> + Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write), + case Res of + '$end_of_table' -> + done; + {[], Cont} -> + %% since `GCBatchsize' is just a "recommendation" for `select', we try only + %% _once_ the continuation and then stop if it yields nothing, to avoid a + %% dead loop. + case mnesia:select(Cont) of + '$end_of_table' -> + done; + {[], _Cont} -> + done; + {DSSessionIds0, _Cont} -> + do_gc_(DSSessionIds0), + more + end; + {DSSessionIds0, _Cont} -> + do_gc_(DSSessionIds0), + more + end + end), + do_gc(Next); +do_gc(done) -> + ok. + +do_gc_(DSSessionIds) -> + lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds), + ?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}), + ok. diff --git a/apps/emqx/src/emqx_persistent_session_ds_sup.erl b/apps/emqx/src/emqx_persistent_session_ds_sup.erl new file mode 100644 index 000000000..5bd620e8b --- /dev/null +++ b/apps/emqx/src/emqx_persistent_session_ds_sup.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_persistent_session_ds_sup). + +-behaviour(supervisor). + +%% API +-export([ + start_link/0 +]). + +%% `supervisor' API +-export([ + init/1 +]). + +%%-------------------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------------------- + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%-------------------------------------------------------------------------------- +%% `supervisor' API +%%-------------------------------------------------------------------------------- + +init(Opts) -> + case emqx_persistent_message:is_persistence_enabled() of + true -> + do_init(Opts); + false -> + ignore + end. + +do_init(_Opts) -> + SupFlags = #{ + strategy => rest_for_one, + intensity => 10, + period => 2, + auto_shutdown => never + }, + CoreChildren = [ + worker(gc_worker, emqx_persistent_session_ds_gc_worker, []) + ], + Children = + case mria_rlog:role() of + core -> CoreChildren; + replicant -> [] + end, + {ok, {SupFlags, Children}}. + +%%-------------------------------------------------------------------------------- +%% Internal fns +%%-------------------------------------------------------------------------------- + +worker(Id, Mod, Args) -> + #{ + id => Id, + start => {Mod, start_link, Args}, + type => worker, + restart => permanent, + shutdown => 10_000, + significant => false + }. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bba55cd97..f46387d3b 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1781,6 +1781,31 @@ fields("session_persistence") -> desc => ?DESC(session_ds_idle_poll_interval) } )}, + {"last_alive_update_interval", + sc( + timeout_duration(), + #{ + default => <<"5000ms">>, + desc => ?DESC(session_ds_last_alive_update_interval) + } + )}, + {"session_gc_interval", + sc( + timeout_duration(), + #{ + default => <<"10m">>, + desc => ?DESC(session_ds_session_gc_interval) + } + )}, + {"session_gc_batch_size", + sc( + pos_integer(), + #{ + default => 100, + importance => ?IMPORTANCE_LOW, + desc => ?DESC(session_ds_session_gc_batch_size) + } + )}, {"force_persistence", sc( boolean(), diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 64ef2e30d..108e8ec09 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -84,7 +84,7 @@ -export([ deliver/3, handle_timeout/3, - disconnect/2, + disconnect/3, terminate/3 ]). @@ -503,10 +503,10 @@ cancel_timer(Name, Timers) -> %%-------------------------------------------------------------------- --spec disconnect(clientinfo(), t()) -> +-spec disconnect(clientinfo(), eqmx_types:conninfo(), t()) -> {idle | shutdown, t()}. -disconnect(_ClientInfo, Session) -> - ?IMPL(Session):disconnect(Session). +disconnect(_ClientInfo, ConnInfo, Session) -> + ?IMPL(Session):disconnect(Session, ConnInfo). -spec terminate(clientinfo(), Reason :: term(), t()) -> ok. diff --git a/apps/emqx/src/emqx_session_mem.erl b/apps/emqx/src/emqx_session_mem.erl index d609435c0..178c71e12 100644 --- a/apps/emqx/src/emqx_session_mem.erl +++ b/apps/emqx/src/emqx_session_mem.erl @@ -87,7 +87,7 @@ deliver/3, replay/3, handle_timeout/3, - disconnect/1, + disconnect/2, terminate/2 ]). @@ -725,8 +725,8 @@ append(L1, L2) -> L1 ++ L2. %%-------------------------------------------------------------------- --spec disconnect(session()) -> {idle, session()}. -disconnect(Session = #session{}) -> +-spec disconnect(session(), emqx_types:conninfo()) -> {idle, session()}. +disconnect(Session = #session{}, _ConnInfo) -> % TODO: isolate expiry timer / timeout handling here? {idle, Session}. diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 79ad9905c..894595f72 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -418,6 +418,9 @@ get_otp_version() -> end. read_otp_version() -> + string:trim(do_read_otp_version()). + +do_read_otp_version() -> ReleasesDir = filename:join([code:root_dir(), "releases"]), Filename = filename:join([ReleasesDir, emqx_app:get_release(), "BUILD_INFO"]), case file:read_file(Filename) of diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 4671851f8..18919103c 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -753,24 +753,15 @@ start_slave(Name, Opts) when is_map(Opts) -> case SlaveMod of ct_slave -> ct:pal("~p: node data dir: ~s", [Node, NodeDataDir]), - ct_slave:start( - Node, - [ - {kill_if_fail, true}, - {monitor_master, true}, - {init_timeout, 20_000}, - {startup_timeout, 20_000}, - {erl_flags, erl_flags()}, - {env, [ - {"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}, - {"EMQX_NODE__COOKIE", Cookie}, - {"EMQX_NODE__DATA_DIR", NodeDataDir} - ]} - ] - ); + Envs = [ + {"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}, + {"EMQX_NODE__COOKIE", Cookie}, + {"EMQX_NODE__DATA_DIR", NodeDataDir} + ], + emqx_cth_peer:start(Node, erl_flags(), Envs); slave -> - Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_", - slave:start_link(host(), Name, ebin_path() ++ Env) + Envs = [{"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}], + emqx_cth_peer:start(Node, ebin_path(), Envs) end end, case DoStart() of @@ -789,13 +780,7 @@ start_slave(Name, Opts) when is_map(Opts) -> %% Node stopping stop_slave(Node0) -> Node = node_name(Node0), - SlaveMod = get_peer_mod(Node), - erase_peer_mod(Node), - case SlaveMod:stop(Node) of - ok -> ok; - {ok, _} -> ok; - {error, not_started, _} -> ok - end. + emqx_cth_peer:stop(Node). %% EPMD starting start_epmd() -> @@ -1022,11 +1007,11 @@ set_envs(Node, Env) -> ). erl_flags() -> - %% One core and redirecting logs to master - "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path(). + %% One core + ["+S", "1:1"] ++ ebin_path(). ebin_path() -> - string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). + ["-pa" | lists:filter(fun is_lib/1, code:get_path())]. is_lib(Path) -> string:prefix(Path, code:lib_dir()) =:= nomatch andalso diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index b41586518..029907f57 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -38,14 +38,14 @@ %% in `end_per_suite/1` or `end_per_group/2`) with the result from step 2. -module(emqx_cth_cluster). --export([start/2]). +-export([start/1, start/2, restart/2]). -export([stop/1, stop_node/1]). --export([start_bare_node/2]). +-export([start_bare_nodes/1, start_bare_nodes/2]). -export([share_load_module/2]). -export([node_name/1, mk_nodespecs/2]). --export([start_apps/2, set_node_opts/2]). +-export([start_apps/2]). -define(APPS_CLUSTERING, [gen_rpc, mria, ekka]). @@ -109,9 +109,12 @@ when }. start(Nodes, ClusterOpts) -> NodeSpecs = mk_nodespecs(Nodes, ClusterOpts), - ct:pal("Starting cluster:\n ~p", [NodeSpecs]), + start(NodeSpecs). + +start(NodeSpecs) -> + ct:pal("(Re)starting nodes:\n ~p", [NodeSpecs]), % 1. Start bare nodes with only basic applications running - _ = emqx_utils:pmap(fun start_node_init/1, NodeSpecs, ?TIMEOUT_NODE_START_MS), + ok = start_nodes_init(NodeSpecs, ?TIMEOUT_NODE_START_MS), % 2. Start applications needed to enable clustering % Generally, this causes some applications to restart, but we deliberately don't % start them yet. @@ -121,6 +124,11 @@ start(Nodes, ClusterOpts) -> _ = emqx_utils:pmap(fun run_node_phase_apps/1, NodeSpecs, ?TIMEOUT_APPS_START_MS), [Node || #{name := Node} <- NodeSpecs]. +restart(Node, Spec) -> + ct:pal("Stopping peer node ~p", [Node]), + ok = emqx_cth_peer:stop(Node), + start([Spec#{boot_type => restart}]). + mk_nodespecs(Nodes, ClusterOpts) -> NodeSpecs = lists:zipwith( fun(N, {Name, Opts}) -> mk_init_nodespec(N, Name, Opts, ClusterOpts) end, @@ -282,8 +290,50 @@ allocate_listener_port(Type, #{base_port := BasePort}) -> allocate_listener_ports(Types, Spec) -> lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]). -start_node_init(Spec = #{name := Node}) -> - Node = start_bare_node(Node, Spec), +start_nodes_init(Specs, Timeout) -> + Names = lists:map(fun(#{name := Name}) -> Name end, Specs), + Nodes = start_bare_nodes(Names, Timeout), + lists:foreach(fun node_init/1, Nodes). + +start_bare_nodes(Names) -> + start_bare_nodes(Names, ?TIMEOUT_NODE_START_MS). +start_bare_nodes(Names, Timeout) -> + Args = erl_flags(), + Envs = [], + Waits = lists:map( + fun(Name) -> + WaitTag = {boot_complete, Name}, + WaitBoot = {self(), WaitTag}, + {ok, _} = emqx_cth_peer:start(Name, Args, Envs, WaitBoot), + WaitTag + end, + Names + ), + Deadline = erlang:monotonic_time() + erlang:convert_time_unit(Timeout, millisecond, nanosecond), + Nodes = wait_boot_complete(Waits, Deadline), + lists:foreach(fun(Node) -> pong = net_adm:ping(Node) end, Nodes), + Nodes. + +wait_boot_complete([], _) -> + []; +wait_boot_complete(Waits, Deadline) -> + case erlang:monotonic_time() > Deadline of + true -> + error({timeout, Waits}); + false -> + ok + end, + receive + {{boot_complete, _Name} = Wait, {started, Node, _Pid}} -> + ct:pal("~p", [Wait]), + [Node | wait_boot_complete(Waits -- [Wait], Deadline)]; + {{boot_complete, _Name}, Otherwise} -> + error({unexpected, Otherwise}) + after 100 -> + wait_boot_complete(Waits, Deadline) + end. + +node_init(Node) -> % Make it possible to call `ct:pal` and friends (if running under rebar3) _ = share_load_module(Node, cthr), % Enable snabbkaffe trace forwarding @@ -300,12 +350,6 @@ run_node_phase_apps(Spec = #{name := Node}) -> ok = start_apps(Node, Spec), ok. -set_node_opts(Node, Spec) -> - erpc:call(Node, persistent_term, put, [{?MODULE, opts}, Spec]). - -get_node_opts(Node) -> - erpc:call(Node, persistent_term, get, [{?MODULE, opts}]). - load_apps(Node, #{apps := Apps}) -> erpc:call(Node, emqx_cth_suite, load_apps, [Apps]). @@ -322,8 +366,12 @@ start_apps(Node, #{apps := Apps} = Spec) -> ok. suite_opts(Spec) -> - maps:with([work_dir], Spec). + maps:with([work_dir, boot_type], Spec). +maybe_join_cluster(_Node, #{boot_type := restart}) -> + %% when restart, the node should already be in the cluster + %% hence no need to (re)join + ok; maybe_join_cluster(_Node, #{role := replicant}) -> ok; maybe_join_cluster(Node, Spec) -> @@ -352,23 +400,7 @@ stop(Nodes) -> stop_node(Name) -> Node = node_name(Name), - try get_node_opts(Node) of - Opts -> - stop_node(Name, Opts) - catch - error:{erpc, _} -> - ok - end. - -stop_node(Node, #{driver := ct_slave}) -> - case ct_slave:stop(Node, [{stop_timeout, ?TIMEOUT_NODE_STOP_S}]) of - {ok, _} -> - ok; - {error, Reason, _} when Reason == not_connected; Reason == not_started -> - ok - end; -stop_node(Node, #{driver := slave}) -> - slave:stop(Node). + ok = emqx_cth_peer:stop(Node). %% Ports @@ -391,36 +423,12 @@ listener_port(BasePort, wss) -> %% --spec start_bare_node(atom(), map()) -> node(). -start_bare_node(Name, Spec = #{driver := ct_slave}) -> - {ok, Node} = ct_slave:start( - node_name(Name), - [ - {kill_if_fail, true}, - {monitor_master, true}, - {init_timeout, 20_000}, - {startup_timeout, 20_000}, - {erl_flags, erl_flags()}, - {env, []} - ] - ), - init_bare_node(Node, Spec); -start_bare_node(Name, Spec = #{driver := slave}) -> - {ok, Node} = slave:start_link(host(), Name, ebin_path()), - init_bare_node(Node, Spec). - -init_bare_node(Node, Spec) -> - pong = net_adm:ping(Node), - % Preserve node spec right on the remote node - ok = set_node_opts(Node, Spec), - Node. - erl_flags() -> - %% One core and redirecting logs to master - "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path(). + %% One core + ["+S", "1:1"] ++ ebin_path(). ebin_path() -> - string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). + ["-pa" | lists:filter(fun is_lib/1, code:get_path())]. is_lib(Path) -> string:prefix(Path, code:lib_dir()) =:= nomatch andalso diff --git a/apps/emqx/test/emqx_cth_peer.erl b/apps/emqx/test/emqx_cth_peer.erl new file mode 100644 index 000000000..8b1996cbd --- /dev/null +++ b/apps/emqx/test/emqx_cth_peer.erl @@ -0,0 +1,79 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Common Test Helper proxy module for slave -> peer migration. +%% OTP 26 has slave module deprecated, use peer instead. + +-module(emqx_cth_peer). + +-export([start/2, start/3, start/4]). +-export([start_link/2, start_link/3, start_link/4]). +-export([stop/1]). + +start(Name, Args) -> + start(Name, Args, []). + +start(Name, Args, Envs) -> + start(Name, Args, Envs, timer:seconds(20)). + +start(Name, Args, Envs, Timeout) when is_atom(Name) -> + do_start(Name, Args, Envs, Timeout, start). + +start_link(Name, Args) -> + start_link(Name, Args, []). + +start_link(Name, Args, Envs) -> + start_link(Name, Args, Envs, timer:seconds(20)). + +start_link(Name, Args, Envs, Timeout) when is_atom(Name) -> + do_start(Name, Args, Envs, Timeout, start_link). + +do_start(Name0, Args, Envs, Timeout, Func) when is_atom(Name0) -> + {Name, Host} = parse_node_name(Name0), + {ok, Pid, Node} = peer:Func(#{ + name => Name, + host => Host, + args => Args, + env => Envs, + wait_boot => Timeout, + longnames => true, + shutdown => {halt, 1000} + }), + true = register(Node, Pid), + {ok, Node}. + +stop(Node) when is_atom(Node) -> + Pid = whereis(Node), + case is_pid(Pid) of + true -> + unlink(Pid), + ok = peer:stop(Pid); + false -> + ct:pal("The control process for node ~p is unexpetedly down", [Node]), + ok + end. + +parse_node_name(NodeName) -> + case string:tokens(atom_to_list(NodeName), "@") of + [Name, Host] -> + {list_to_atom(Name), Host}; + [_] -> + {NodeName, host()} + end. + +host() -> + [_Name, Host] = string:tokens(atom_to_list(node()), "@"), + Host. diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 401d4f59d..5e91b92c9 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -453,6 +453,9 @@ stop_apps(Apps) -> %% +verify_clean_suite_state(#{boot_type := restart}) -> + %% when testing node restart, we do not need to verify clean state + ok; verify_clean_suite_state(#{work_dir := WorkDir}) -> {ok, []} = file:list_dir(WorkDir), false = emqx_schema_hooks:any_injections(), diff --git a/apps/emqx/test/emqx_metrics_worker_SUITE.erl b/apps/emqx/test/emqx_metrics_worker_SUITE.erl index 194c9cc99..784eac18e 100644 --- a/apps/emqx/test/emqx_metrics_worker_SUITE.erl +++ b/apps/emqx/test/emqx_metrics_worker_SUITE.erl @@ -53,9 +53,9 @@ t_get_metrics(_) -> ?assertMatch( #{ rate := #{ - a := #{current := 0.0, max := 0.0, last5m := 0.0}, - b := #{current := 0.0, max := 0.0, last5m := 0.0}, - c := #{current := 0.0, max := 0.0, last5m := 0.0} + a := #{current := +0.0, max := +0.0, last5m := +0.0}, + b := #{current := +0.0, max := +0.0, last5m := +0.0}, + c := #{current := +0.0, max := +0.0, last5m := +0.0} }, gauges := #{}, counters := #{ @@ -118,9 +118,9 @@ t_clear_metrics(_Config) -> ?assertMatch( #{ rate := #{ - a := #{current := 0.0, max := 0.0, last5m := 0.0}, - b := #{current := 0.0, max := 0.0, last5m := 0.0}, - c := #{current := 0.0, max := 0.0, last5m := 0.0} + a := #{current := +0.0, max := +0.0, last5m := +0.0}, + b := #{current := +0.0, max := +0.0, last5m := +0.0}, + c := #{current := +0.0, max := +0.0, last5m := +0.0} }, gauges := #{}, slides := #{}, @@ -145,7 +145,7 @@ t_clear_metrics(_Config) -> #{ counters => #{}, gauges => #{}, - rate => #{current => 0.0, last5m => 0.0, max => 0.0}, + rate => #{current => +0.0, last5m => +0.0, max => +0.0}, slides => #{} }, emqx_metrics_worker:get_metrics(?NAME, Id) @@ -160,9 +160,9 @@ t_reset_metrics(_) -> ?assertMatch( #{ rate := #{ - a := #{current := 0.0, max := 0.0, last5m := 0.0}, - b := #{current := 0.0, max := 0.0, last5m := 0.0}, - c := #{current := 0.0, max := 0.0, last5m := 0.0} + a := #{current := +0.0, max := +0.0, last5m := +0.0}, + b := #{current := +0.0, max := +0.0, last5m := +0.0}, + c := #{current := +0.0, max := +0.0, last5m := +0.0} }, gauges := #{}, counters := #{ diff --git a/apps/emqx/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl index 0bfde981c..1d9539409 100644 --- a/apps/emqx/test/emqx_mountpoint_SUITE.erl +++ b/apps/emqx/test/emqx_mountpoint_SUITE.erl @@ -58,9 +58,6 @@ t_mount_share(_) -> TopicFilters = [T], ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), - %% should not mount share topic when make message. - Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), - ?assertEqual( TopicFilter, mount(undefined, TopicFilter) @@ -89,8 +86,6 @@ t_unmount_share(_) -> ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), - %% should not unmount share topic when make message. - Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), ?assertEqual( TopicFilter, unmount(undefined, TopicFilter) diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index f8f7baaf1..80a83c0a4 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -233,7 +233,7 @@ t_session_subscription_iterators(Config) -> ), ok. -t_qos0(Config) -> +t_qos0(_Config) -> Sub = connect(<>, true, 30), Pub = connect(<>, true, 0), try @@ -258,7 +258,7 @@ t_qos0(Config) -> emqtt:stop(Pub) end. -t_publish_as_persistent(Config) -> +t_publish_as_persistent(_Config) -> Sub = connect(<>, true, 30), Pub = connect(<>, true, 30), try @@ -272,9 +272,8 @@ t_publish_as_persistent(Config) -> ?assertMatch( [ #{qos := 0, topic := <<"t/1">>, payload := <<"1">>}, - #{qos := 1, topic := <<"t/1">>, payload := <<"2">>} - %% TODO: QoS 2 - %% #{qos := 2, topic := <<"t/1">>, payload := <<"3">>} + #{qos := 1, topic := <<"t/1">>, payload := <<"2">>}, + #{qos := 2, topic := <<"t/1">>, payload := <<"3">>} ], receive_messages(3) ) diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index 1be929c7f..66bb8dcf5 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -17,6 +17,7 @@ -module(emqx_persistent_session_SUITE). -include_lib("stdlib/include/assert.hrl"). +-include_lib("emqx/include/asserts.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -53,10 +54,10 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), TCsNonGeneric = [t_choose_impl], + TCGroups = [{group, tcp}, {group, quic}, {group, ws}], [ - {persistence_disabled, [{group, no_kill_connection_process}]}, - {persistence_enabled, [{group, no_kill_connection_process}]}, - {no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]}, + {persistence_disabled, TCGroups}, + {persistence_enabled, TCGroups}, {tcp, [], TCs}, {quic, [], TCs -- TCsNonGeneric}, {ws, [], TCs -- TCsNonGeneric} @@ -74,7 +75,7 @@ init_per_group(persistence_enabled, Config) -> {persistence, ds} | Config ]; -init_per_group(Group, Config) when Group == tcp -> +init_per_group(tcp, Config) -> Apps = emqx_cth_suite:start( [{emqx, ?config(emqx_config, Config)}], #{work_dir => emqx_cth_suite:work_dir(Config)} @@ -85,7 +86,7 @@ init_per_group(Group, Config) when Group == tcp -> {group_apps, Apps} | Config ]; -init_per_group(Group, Config) when Group == ws -> +init_per_group(ws, Config) -> Apps = emqx_cth_suite:start( [{emqx, ?config(emqx_config, Config)}], #{work_dir => emqx_cth_suite:work_dir(Config)} @@ -99,7 +100,7 @@ init_per_group(Group, Config) when Group == ws -> {group_apps, Apps} | Config ]; -init_per_group(Group, Config) when Group == quic -> +init_per_group(quic, Config) -> Apps = emqx_cth_suite:start( [ {emqx, @@ -118,11 +119,7 @@ init_per_group(Group, Config) when Group == quic -> {ssl, true}, {group_apps, Apps} | Config - ]; -init_per_group(no_kill_connection_process, Config) -> - [{kill_connection_process, false} | Config]; -init_per_group(kill_connection_process, Config) -> - [{kill_connection_process, true} | Config]. + ]. get_listener_port(Type, Name) -> case emqx_config:get([listeners, Type, Name, bind]) of @@ -194,6 +191,8 @@ receive_message_loop(Count, Deadline) -> receive {publish, Msg} -> [Msg | receive_message_loop(Count - 1, Deadline)]; + {pubrel, Msg} -> + [{pubrel, Msg} | receive_message_loop(Count - 1, Deadline)]; _Other -> receive_message_loop(Count, Deadline) after Timeout -> @@ -201,39 +200,62 @@ receive_message_loop(Count, Deadline) -> end. maybe_kill_connection_process(ClientId, Config) -> - case ?config(kill_connection_process, Config) of - true -> - case emqx_cm:lookup_channels(ClientId) of - [] -> - ok; - [ConnectionPid] -> - ?assert(is_pid(ConnectionPid)), - Ref = monitor(process, ConnectionPid), - ConnectionPid ! die_if_test, - receive - {'DOWN', Ref, process, ConnectionPid, normal} -> ok - after 3000 -> error(process_did_not_die) - end, - wait_for_cm_unregister(ClientId) - end; - false -> - ok - end. - -wait_for_cm_unregister(ClientId) -> - wait_for_cm_unregister(ClientId, 100). - -wait_for_cm_unregister(_ClientId, 0) -> - error(cm_did_not_unregister); -wait_for_cm_unregister(ClientId, N) -> + Persistence = ?config(persistence, Config), case emqx_cm:lookup_channels(ClientId) of [] -> ok; - [_] -> - timer:sleep(100), - wait_for_cm_unregister(ClientId, N - 1) + [ConnectionPid] when Persistence == ds -> + Ref = monitor(process, ConnectionPid), + ConnectionPid ! die_if_test, + ?assertReceive( + {'DOWN', Ref, process, ConnectionPid, Reason} when + Reason == normal orelse Reason == noproc, + 3000 + ), + wait_connection_process_unregistered(ClientId); + _ -> + ok end. +wait_connection_process_dies(ClientId) -> + case emqx_cm:lookup_channels(ClientId) of + [] -> + ok; + [ConnectionPid] -> + Ref = monitor(process, ConnectionPid), + ?assertReceive( + {'DOWN', Ref, process, ConnectionPid, Reason} when + Reason == normal orelse Reason == noproc, + 3000 + ), + wait_connection_process_unregistered(ClientId) + end. + +wait_connection_process_unregistered(ClientId) -> + ?retry( + _Timeout = 100, + _Retries = 20, + ?assertEqual([], emqx_cm:lookup_channels(ClientId)) + ). + +wait_channel_disconnected(ClientId) -> + ?retry( + _Timeout = 100, + _Retries = 20, + case emqx_cm:lookup_channels(ClientId) of + [] -> + false; + [ChanPid] -> + false = emqx_cm:is_channel_connected(ChanPid) + end + ). + +disconnect_client(ClientPid) -> + ClientId = proplists:get_value(clientid, emqtt:info(ClientPid)), + ok = emqtt:disconnect(ClientPid), + false = wait_channel_disconnected(ClientId), + ok. + messages(Topic, Payloads) -> messages(Topic, Payloads, ?QOS_2). @@ -272,23 +294,7 @@ do_publish(Messages = [_ | _], PublishFun, WaitForUnregister) -> lists:foreach(fun(Message) -> PublishFun(Client, Message) end, Messages), ok = emqtt:disconnect(Client), %% Snabbkaffe sometimes fails unless all processes are gone. - case WaitForUnregister of - false -> - ok; - true -> - case emqx_cm:lookup_channels(ClientID) of - [] -> - ok; - [ConnectionPid] -> - ?assert(is_pid(ConnectionPid)), - Ref1 = monitor(process, ConnectionPid), - receive - {'DOWN', Ref1, process, ConnectionPid, _} -> ok - after 3000 -> error(process_did_not_die) - end, - wait_for_cm_unregister(ClientID) - end - end + WaitForUnregister andalso wait_connection_process_dies(ClientID) end ), receive @@ -347,8 +353,6 @@ t_connect_discards_existing_client(Config) -> end. %% [MQTT-3.1.2-23] -t_connect_session_expiry_interval(init, Config) -> skip_ds_tc(Config); -t_connect_session_expiry_interval('end', _Config) -> ok. t_connect_session_expiry_interval(Config) -> ConnFun = ?config(conn_fun, Config), Topic = ?config(topic, Config), @@ -356,6 +360,45 @@ t_connect_session_expiry_interval(Config) -> Payload = <<"test message">>, ClientId = ?config(client_id, Config), + {ok, Client1} = emqtt:start_link([ + {clientid, ClientId}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 30}} + | Config + ]), + {ok, _} = emqtt:ConnFun(Client1), + {ok, _, [?RC_GRANTED_QOS_1]} = emqtt:subscribe(Client1, STopic, ?QOS_1), + ok = emqtt:disconnect(Client1), + + maybe_kill_connection_process(ClientId, Config), + + publish(Topic, Payload, ?QOS_1), + + {ok, Client2} = emqtt:start_link([ + {clientid, ClientId}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 30}}, + {clean_start, false} + | Config + ]), + {ok, _} = emqtt:ConnFun(Client2), + [Msg | _] = receive_messages(1), + ?assertEqual({ok, iolist_to_binary(Topic)}, maps:find(topic, Msg)), + ?assertEqual({ok, iolist_to_binary(Payload)}, maps:find(payload, Msg)), + ?assertEqual({ok, ?QOS_1}, maps:find(qos, Msg)), + ok = emqtt:disconnect(Client2). + +%% [MQTT-3.1.2-23] +%% TODO: un-skip after QoS 2 support is implemented in DS. +t_connect_session_expiry_interval_qos2(init, Config) -> skip_ds_tc(Config); +t_connect_session_expiry_interval_qos2('end', _Config) -> ok. +t_connect_session_expiry_interval_qos2(Config) -> + ConnFun = ?config(conn_fun, Config), + Topic = ?config(topic, Config), + STopic = ?config(stopic, Config), + Payload = <<"test message">>, + ClientId = ?config(client_id, Config), + {ok, Client1} = emqtt:start_link([ {clientid, ClientId}, {proto_ver, v5}, @@ -438,7 +481,7 @@ t_cancel_on_disconnect(Config) -> {ok, _} = emqtt:ConnFun(Client1), ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 0}), - wait_for_cm_unregister(ClientId), + wait_connection_process_unregistered(ClientId), {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, @@ -470,7 +513,7 @@ t_persist_on_disconnect(Config) -> %% Strangely enough, the disconnect is reported as successful by emqtt. ok = emqtt:disconnect(Client1, 0, #{'Session-Expiry-Interval' => 30}), - wait_for_cm_unregister(ClientId), + wait_connection_process_unregistered(ClientId), {ok, Client2} = emqtt:start_link([ {clientid, ClientId}, @@ -485,8 +528,6 @@ t_persist_on_disconnect(Config) -> ?assertEqual(0, client_info(session_present, Client2)), ok = emqtt:disconnect(Client2). -t_process_dies_session_expires(init, Config) -> skip_ds_tc(Config); -t_process_dies_session_expires('end', _Config) -> ok. t_process_dies_session_expires(Config) -> %% Emulate an error in the connect process, %% or that the node of the process goes down. @@ -582,7 +623,7 @@ t_publish_many_while_client_is_gone_qos1(Config) -> {clientid, ClientId}, {properties, #{'Session-Expiry-Interval' => 30}}, {clean_start, true}, - {auto_ack, false} + {auto_ack, never} | Config ]), {ok, _} = emqtt:ConnFun(Client1), @@ -629,8 +670,7 @@ t_publish_many_while_client_is_gone_qos1(Config) -> ?assertEqual( get_topicwise_order(Pubs1), - get_topicwise_order(Msgs1), - Msgs1 + get_topicwise_order(Msgs1) ), NAcked = 4, @@ -639,7 +679,7 @@ t_publish_many_while_client_is_gone_qos1(Config) -> %% Ensure that PUBACKs are propagated to the channel. pong = emqtt:ping(Client1), - ok = emqtt:disconnect(Client1), + ok = disconnect_client(Client1), maybe_kill_connection_process(ClientId, Config), Pubs2 = [ @@ -686,23 +726,8 @@ t_publish_many_while_client_is_gone_qos1(Config) -> [maps:with([packet_id, topic, payload], M) || M <- lists:sublist(Msgs2, NSame)] ), - ok = emqtt:disconnect(Client2). + ok = disconnect_client(Client2). -get_topicwise_order(Msgs) -> - maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs). - -get_msgpub_topic(#mqtt_msg{topic = Topic}) -> - Topic; -get_msgpub_topic(#{topic := Topic}) -> - Topic. - -get_msgpub_payload(#mqtt_msg{payload = Payload}) -> - Payload; -get_msgpub_payload(#{payload := Payload}) -> - Payload. - -t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config); -t_publish_while_client_is_gone('end', _Config) -> ok. t_publish_while_client_is_gone(Config) -> %% A persistent session should receive messages in its %% subscription even if the process owning the session dies. @@ -745,6 +770,157 @@ t_publish_while_client_is_gone(Config) -> ok = emqtt:disconnect(Client2). +t_publish_many_while_client_is_gone(Config) -> + %% A persistent session should receive all of the still unacked messages + %% for its subscriptions after the client dies or reconnects, in addition + %% to PUBRELs for the messages it has PUBRECed. While client must send + %% PUBACKs and PUBRECs in order, those orders are independent of each other. + ClientId = ?config(client_id, Config), + ConnFun = ?config(conn_fun, Config), + ClientOpts = [ + {proto_ver, v5}, + {clientid, ClientId}, + {properties, #{'Session-Expiry-Interval' => 30}}, + {auto_ack, never} + | Config + ], + + {ok, Client1} = emqtt:start_link([{clean_start, true} | ClientOpts]), + {ok, _} = emqtt:ConnFun(Client1), + {ok, _, [?QOS_1]} = emqtt:subscribe(Client1, <<"t/+/foo">>, ?QOS_1), + {ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"msg/feed/#">>, ?QOS_2), + {ok, _, [?QOS_2]} = emqtt:subscribe(Client1, <<"loc/+/+/+">>, ?QOS_2), + + Pubs1 = [ + #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M1">>, qos = 1}, + #mqtt_msg{topic = <<"t/42/foo">>, payload = <<"M2">>, qos = 1}, + #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M3">>, qos = 2}, + #mqtt_msg{topic = <<"loc/1/2/42">>, payload = <<"M4">>, qos = 2}, + #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M5">>, qos = 2}, + #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M6">>, qos = 1}, + #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M7">>, qos = 2}, + #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M8">>, qos = 1}, + #mqtt_msg{topic = <<"msg/feed/me">>, payload = <<"M9">>, qos = 2} + ], + ok = publish_many(Pubs1), + NPubs1 = length(Pubs1), + + Msgs1 = receive_messages(NPubs1), + ct:pal("Msgs1 = ~p", [Msgs1]), + NMsgs1 = length(Msgs1), + ?assertEqual(NPubs1, NMsgs1), + + ?assertEqual( + get_topicwise_order(Pubs1), + get_topicwise_order(Msgs1) + ), + + %% PUBACK every QoS 1 message. + lists:foreach( + fun(PktId) -> ok = emqtt:puback(Client1, PktId) end, + [PktId || #{qos := 1, packet_id := PktId} <- Msgs1] + ), + + %% PUBREC first `NRecs` QoS 2 messages. + NRecs = 3, + PubRecs1 = lists:sublist([PktId || #{qos := 2, packet_id := PktId} <- Msgs1], NRecs), + lists:foreach( + fun(PktId) -> ok = emqtt:pubrec(Client1, PktId) end, + PubRecs1 + ), + + %% Ensure that PUBACKs / PUBRECs are propagated to the channel. + pong = emqtt:ping(Client1), + + %% Receive PUBRELs for the sent PUBRECs. + PubRels1 = receive_messages(NRecs), + ct:pal("PubRels1 = ~p", [PubRels1]), + ?assertEqual( + PubRecs1, + [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1], + PubRels1 + ), + + ok = disconnect_client(Client1), + maybe_kill_connection_process(ClientId, Config), + + Pubs2 = [ + #mqtt_msg{topic = <<"loc/3/4/5">>, payload = <<"M10">>, qos = 2}, + #mqtt_msg{topic = <<"t/100/foo">>, payload = <<"M11">>, qos = 1}, + #mqtt_msg{topic = <<"msg/feed/friend">>, payload = <<"M12">>, qos = 2} + ], + ok = publish_many(Pubs2), + NPubs2 = length(Pubs2), + + {ok, Client2} = emqtt:start_link([{clean_start, false} | ClientOpts]), + {ok, _} = emqtt:ConnFun(Client2), + + %% Try to receive _at most_ `NPubs` messages. + %% There shouldn't be that much unacked messages in the replay anyway, + %% but it's an easy number to pick. + NPubs = NPubs1 + NPubs2, + Msgs2 = receive_messages(NPubs, _Timeout = 2000), + ct:pal("Msgs2 = ~p", [Msgs2]), + + %% We should again receive PUBRELs for the PUBRECs we sent earlier. + ?assertEqual( + get_msgs_essentials(PubRels1), + [get_msg_essentials(PubRel) || PubRel = {pubrel, _} <- Msgs2] + ), + + %% We should receive duplicates only for QoS 2 messages where PUBRELs were + %% not sent, in the same order as the original messages. + Msgs2Dups = [get_msg_essentials(M) || M = #{dup := true} <- Msgs2], + ?assertEqual( + Msgs2Dups, + [M || M = #{qos := 2} <- Msgs2Dups] + ), + ?assertEqual( + get_msgs_essentials(pick_respective_msgs(Msgs2Dups, Msgs1)), + Msgs2Dups + ), + + %% Now complete all yet incomplete QoS 2 message flows instead. + PubRecs2 = [PktId || #{qos := 2, packet_id := PktId} <- Msgs2], + lists:foreach( + fun(PktId) -> ok = emqtt:pubrec(Client2, PktId) end, + PubRecs2 + ), + + PubRels2 = receive_messages(length(PubRecs2)), + ct:pal("PubRels2 = ~p", [PubRels2]), + ?assertEqual( + PubRecs2, + [PktId || {pubrel, #{packet_id := PktId}} <- PubRels2], + PubRels2 + ), + + %% PUBCOMP every PUBREL. + PubComps = [PktId || {pubrel, #{packet_id := PktId}} <- PubRels1 ++ PubRels2], + lists:foreach( + fun(PktId) -> ok = emqtt:pubcomp(Client2, PktId) end, + PubComps + ), + + %% Ensure that PUBCOMPs are propagated to the channel. + pong = emqtt:ping(Client2), + + ok = disconnect_client(Client2), + maybe_kill_connection_process(ClientId, Config), + + {ok, Client3} = emqtt:start_link([{clean_start, false} | ClientOpts]), + {ok, _} = emqtt:ConnFun(Client3), + + %% Only the last unacked QoS 1 message should be retransmitted. + Msgs3 = receive_messages(NPubs, _Timeout = 2000), + ct:pal("Msgs3 = ~p", [Msgs3]), + ?assertMatch( + [#{topic := <<"t/100/foo">>, payload := <<"M11">>, qos := 1, dup := true}], + Msgs3 + ), + + ok = disconnect_client(Client3). + t_clean_start_drops_subscriptions(Config) -> %% 1. A persistent session is started and disconnected. %% 2. While disconnected, a message is published and persisted. @@ -795,6 +971,7 @@ t_clean_start_drops_subscriptions(Config) -> [Msg1] = receive_messages(1), ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, Msg1)), + pong = emqtt:ping(Client2), ok = emqtt:disconnect(Client2), maybe_kill_connection_process(ClientId, Config), @@ -812,6 +989,7 @@ t_clean_start_drops_subscriptions(Config) -> [Msg2] = receive_messages(1), ?assertEqual({ok, iolist_to_binary(Payload3)}, maps:find(payload, Msg2)), + pong = emqtt:ping(Client3), ok = emqtt:disconnect(Client3). t_unsubscribe(Config) -> @@ -875,6 +1053,30 @@ t_multiple_subscription_matches(Config) -> ?assertEqual({ok, 2}, maps:find(qos, Msg2)), ok = emqtt:disconnect(Client2). +get_topicwise_order(Msgs) -> + maps:groups_from_list(fun get_msgpub_topic/1, fun get_msgpub_payload/1, Msgs). + +get_msgpub_topic(#mqtt_msg{topic = Topic}) -> + Topic; +get_msgpub_topic(#{topic := Topic}) -> + Topic. + +get_msgpub_payload(#mqtt_msg{payload = Payload}) -> + Payload; +get_msgpub_payload(#{payload := Payload}) -> + Payload. + +get_msg_essentials(Msg = #{}) -> + maps:with([packet_id, topic, payload, qos], Msg); +get_msg_essentials({pubrel, Msg}) -> + {pubrel, maps:with([packet_id, reason_code], Msg)}. + +get_msgs_essentials(Msgs) -> + [get_msg_essentials(M) || M <- Msgs]. + +pick_respective_msgs(MsgRefs, Msgs) -> + [M || M <- Msgs, Ref <- MsgRefs, maps:get(packet_id, M) =:= maps:get(packet_id, Ref)]. + skip_ds_tc(Config) -> case ?config(persistence, Config) of ds -> diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 8fe052af8..c16277884 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -80,7 +80,7 @@ t_mnesia(_) -> ct:sleep(200). t_cleanup_membership_mnesia_down(_Config) -> - Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME), + Slave = emqx_cth_cluster:node_name(node2), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), @@ -92,7 +92,7 @@ t_cleanup_membership_mnesia_down(_Config) -> ?assertEqual([<<"d/e/f">>], emqx_router:topics()). t_cleanup_membership_node_down(_Config) -> - Slave = emqx_cth_cluster:node_name(?FUNCTION_NAME), + Slave = emqx_cth_cluster:node_name(node3), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), @@ -104,7 +104,7 @@ t_cleanup_membership_node_down(_Config) -> ?assertEqual([<<"d/e/f">>], emqx_router:topics()). t_cleanup_monitor_node_down(_Config) -> - Slave = emqx_cth_cluster:start_bare_node(?FUNCTION_NAME, #{driver => ct_slave}), + [Slave] = emqx_cth_cluster:start_bare_nodes([node4]), emqx_router:add_route(<<"a/b/c">>, Slave), emqx_router:add_route(<<"d/e/f">>, node()), ?assertMatch([_, _], emqx_router:topics()), diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index a54e1b4dd..c9ad63cf1 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -218,38 +218,41 @@ t_routing_schema_switch(VFrom, VTo, Config) -> ], #{work_dir => WorkDir} ), - % Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively Nodes = [Node1, Node2, Node3], - ?assertEqual( - [{ok, VTo}, {ok, VTo}, {ok, VTo}], - erpc:multicall(Nodes, emqx_router, get_schema_vsn, []) - ), - % Wait for all nodes to agree on cluster state - ?retry( - 500, - 10, - ?assertMatch( - [{ok, [Node1, Node2, Node3]}], - lists:usort(erpc:multicall(Nodes, emqx, running_nodes, [])) - ) - ), - % Verify that routing works as expected - C2 = start_client(Node2), - ok = subscribe(C2, <<"a/+/d">>), - C3 = start_client(Node3), - ok = subscribe(C3, <<"d/e/f/#">>), - {ok, _} = publish(C1, <<"a/b/d">>, <<"hey-newbies">>), - {ok, _} = publish(C2, <<"a/b/c">>, <<"hi">>), - {ok, _} = publish(C3, <<"d/e/f/42">>, <<"hello">>), - ?assertReceive({pub, C2, #{topic := <<"a/b/d">>, payload := <<"hey-newbies">>}}), - ?assertReceive({pub, C1, #{topic := <<"a/b/c">>, payload := <<"hi">>}}), - ?assertReceive({pub, C1, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), - ?assertReceive({pub, C3, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), - ?assertNotReceive(_), - ok = emqtt:stop(C1), - ok = emqtt:stop(C2), - ok = emqtt:stop(C3), - ok = emqx_cth_cluster:stop(Nodes). + try + % Verify that new nodes switched to schema v1/v2 in presence of v1/v2 routes respectively + ?assertEqual( + [{ok, VTo}, {ok, VTo}, {ok, VTo}], + erpc:multicall(Nodes, emqx_router, get_schema_vsn, []) + ), + % Wait for all nodes to agree on cluster state + ?retry( + 500, + 10, + ?assertMatch( + [{ok, [Node1, Node2, Node3]}], + lists:usort(erpc:multicall(Nodes, emqx, running_nodes, [])) + ) + ), + % Verify that routing works as expected + C2 = start_client(Node2), + ok = subscribe(C2, <<"a/+/d">>), + C3 = start_client(Node3), + ok = subscribe(C3, <<"d/e/f/#">>), + {ok, _} = publish(C1, <<"a/b/d">>, <<"hey-newbies">>), + {ok, _} = publish(C2, <<"a/b/c">>, <<"hi">>), + {ok, _} = publish(C3, <<"d/e/f/42">>, <<"hello">>), + ?assertReceive({pub, C2, #{topic := <<"a/b/d">>, payload := <<"hey-newbies">>}}), + ?assertReceive({pub, C1, #{topic := <<"a/b/c">>, payload := <<"hi">>}}), + ?assertReceive({pub, C1, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), + ?assertReceive({pub, C3, #{topic := <<"d/e/f/42">>, payload := <<"hello">>}}), + ?assertNotReceive(_), + ok = emqtt:stop(C1), + ok = emqtt:stop(C2), + ok = emqtt:stop(C3) + after + ok = emqx_cth_cluster:stop(Nodes) + end. %% diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 86887eff0..cc6908fb6 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -63,6 +63,7 @@ init_per_suite(Config) -> end, emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), + emqx_logger:set_log_level(debug), [{dist_pid, DistPid} | Config]. end_per_suite(Config) -> @@ -574,7 +575,7 @@ t_local(Config) when is_list(Config) -> <<"sticky_group">> => sticky }, - Node = start_slave('local_shared_sub_testtesttest', 21999), + Node = start_slave('local_shared_sub_local_1', 21999), ok = ensure_group_config(GroupConfig), ok = ensure_group_config(Node, GroupConfig), @@ -627,7 +628,7 @@ t_remote(Config) when is_list(Config) -> <<"sticky_group">> => sticky }, - Node = start_slave('remote_shared_sub_testtesttest', 21999), + Node = start_slave('remote_shared_sub_remote_1', 21999), ok = ensure_group_config(GroupConfig), ok = ensure_group_config(Node, GroupConfig), @@ -676,7 +677,7 @@ t_local_fallback(Config) when is_list(Config) -> Topic = <<"local_foo/bar">>, ClientId1 = <<"ClientId1">>, ClientId2 = <<"ClientId2">>, - Node = start_slave('local_fallback_shared_sub_test', 11888), + Node = start_slave('local_fallback_shared_sub_1', 11888), {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}]), {ok, _} = emqtt:connect(ConnPid1), @@ -1253,34 +1254,24 @@ recv_msgs(Count, Msgs) -> end. start_slave(Name, Port) -> - {ok, Node} = ct_slave:start( - list_to_atom(atom_to_list(Name) ++ "@" ++ host()), - [ - {kill_if_fail, true}, - {monitor_master, true}, - {init_timeout, 10000}, - {startup_timeout, 10000}, - {erl_flags, ebin_path()} - ] + {ok, Node} = emqx_cth_peer:start_link( + Name, + ebin_path() ), - pong = net_adm:ping(Node), setup_node(Node, Port), Node. stop_slave(Node) -> rpc:call(Node, mria, leave, []), - ct_slave:stop(Node). + emqx_cth_peer:stop(Node). host() -> [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. ebin_path() -> - string:join(["-pa" | lists:filter(fun is_lib/1, code:get_path())], " "). - -is_lib(Path) -> - string:prefix(Path, code:lib_dir()) =:= nomatch. + ["-pa" | code:get_path()]. setup_node(Node, Port) -> EnvHandler = diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 3d4d5f467..d84d6ff81 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_ldap, [ {description, "EMQX LDAP Authentication and Authorization"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_auth_ldap_app, []}}, {applications, [ diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index bd33606d3..b5669e706 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_redis, [ {description, "EMQX Redis Authentication and Authorization"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_auth_redis_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_action_info.erl b/apps/emqx_bridge/src/emqx_action_info.erl index ddb8424ae..f975a1c93 100644 --- a/apps/emqx_bridge/src/emqx_action_info.erl +++ b/apps/emqx_bridge/src/emqx_action_info.erl @@ -90,7 +90,7 @@ hard_coded_action_info_modules_ee() -> -endif. hard_coded_action_info_modules_common() -> - []. + [emqx_bridge_http_action_info]. hard_coded_action_info_modules() -> hard_coded_action_info_modules_common() ++ hard_coded_action_info_modules_ee(). diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index f829b12df..2aa610f24 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.30"}, + {vsn, "0.1.31"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 9e14c0c9a..d26a44a1d 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -364,7 +364,7 @@ get_metrics(Type, Name) -> maybe_upgrade(mqtt, Config) -> emqx_bridge_compatible_config:maybe_upgrade(Config); maybe_upgrade(webhook, Config) -> - emqx_bridge_compatible_config:webhook_maybe_upgrade(Config); + emqx_bridge_compatible_config:http_maybe_upgrade(Config); maybe_upgrade(_Other, Config) -> Config. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index a3c058abb..b725eb740 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -143,7 +143,7 @@ param_path_id() -> #{ in => path, required => true, - example => <<"webhook:webhook_example">>, + example => <<"http:http_example">>, desc => ?DESC("desc_param_path_id") } )}. @@ -166,9 +166,9 @@ bridge_info_array_example(Method) -> bridge_info_examples(Method) -> maps:merge( #{ - <<"webhook_example">> => #{ - summary => <<"WebHook">>, - value => info_example(webhook, Method) + <<"http_example">> => #{ + summary => <<"HTTP">>, + value => info_example(http, Method) }, <<"mqtt_example">> => #{ summary => <<"MQTT Bridge">>, @@ -201,7 +201,7 @@ method_example(Type, Method) when Method == get; Method == post -> method_example(_Type, put) -> #{}. -info_example_basic(webhook) -> +info_example_basic(http) -> #{ enable => true, url => <<"http://localhost:9901/messages/${topic}">>, @@ -212,7 +212,7 @@ info_example_basic(webhook) -> pool_size => 4, enable_pipelining => 100, ssl => #{enable => false}, - local_topic => <<"emqx_webhook/#">>, + local_topic => <<"emqx_http/#">>, method => post, body => <<"${payload}">>, resource_opts => #{ @@ -650,7 +650,8 @@ create_or_update_bridge(BridgeType0, BridgeName, Conf, HttpStatusCode) -> get_metrics_from_local_node(BridgeType0, BridgeName) -> BridgeType = upgrade_type(BridgeType0), - format_metrics(emqx_bridge:get_metrics(BridgeType, BridgeName)). + MetricsResult = emqx_bridge:get_metrics(BridgeType, BridgeName), + format_metrics(MetricsResult). '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> ?TRY_PARSE_ID( diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 231548f30..0a870abb8 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -63,18 +63,23 @@ ). -if(?EMQX_RELEASE_EDITION == ee). -bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; -bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; -bridge_to_resource_type(<<"webhook">>) -> emqx_bridge_http_connector; -bridge_to_resource_type(webhook) -> emqx_bridge_http_connector; -bridge_to_resource_type(BridgeType) -> emqx_bridge_enterprise:resource_type(BridgeType). +bridge_to_resource_type(BridgeType) when is_binary(BridgeType) -> + bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8)); +bridge_to_resource_type(mqtt) -> + emqx_bridge_mqtt_connector; +bridge_to_resource_type(webhook) -> + emqx_bridge_http_connector; +bridge_to_resource_type(BridgeType) -> + emqx_bridge_enterprise:resource_type(BridgeType). bridge_impl_module(BridgeType) -> emqx_bridge_enterprise:bridge_impl_module(BridgeType). -else. -bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; -bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; -bridge_to_resource_type(<<"webhook">>) -> emqx_bridge_http_connector; -bridge_to_resource_type(webhook) -> emqx_bridge_http_connector. +bridge_to_resource_type(BridgeType) when is_binary(BridgeType) -> + bridge_to_resource_type(binary_to_existing_atom(BridgeType, utf8)); +bridge_to_resource_type(mqtt) -> + emqx_bridge_mqtt_connector; +bridge_to_resource_type(webhook) -> + emqx_bridge_http_connector. bridge_impl_module(_BridgeType) -> undefined. -endif. @@ -309,6 +314,7 @@ remove(Type, Name, _Conf, _Opts) -> emqx_resource:remove_local(resource_id(Type, Name)). %% convert bridge configs to what the connector modules want +%% TODO: remove it, if the http_bridge already ported to v2 parse_confs( <<"webhook">>, _Name, diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 1863ed84b..97d0afb43 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -1188,7 +1188,7 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> %% If the bridge v2 does not exist, it is a valid bridge v1 PreviousRawConf = undefined, split_bridge_v1_config_and_create_helper( - BridgeV1Type, BridgeName, RawConf, PreviousRawConf + BridgeV1Type, BridgeName, RawConf, PreviousRawConf, fun() -> ok end ); _Conf -> case ?MODULE:bridge_v1_is_valid(BridgeV1Type, BridgeName) of @@ -1198,9 +1198,13 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> PreviousRawConf = emqx:get_raw_config( [?ROOT_KEY, BridgeV2Type, BridgeName], undefined ), - bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps), + %% To avoid losing configurations. We have to make sure that no crash occurs + %% during deletion and creation of configurations. + PreCreateFun = fun() -> + bridge_v1_check_deps_and_remove(BridgeV1Type, BridgeName, RemoveDeps) + end, split_bridge_v1_config_and_create_helper( - BridgeV1Type, BridgeName, RawConf, PreviousRawConf + BridgeV1Type, BridgeName, RawConf, PreviousRawConf, PreCreateFun ); false -> %% If the bridge v2 exists, it is not a valid bridge v1 @@ -1208,16 +1212,49 @@ bridge_v1_split_config_and_create(BridgeV1Type, BridgeName, RawConf) -> end end. -split_bridge_v1_config_and_create_helper(BridgeV1Type, BridgeName, RawConf, PreviousRawConf) -> - #{ - connector_type := ConnectorType, - connector_name := NewConnectorName, - connector_conf := NewConnectorRawConf, - bridge_v2_type := BridgeType, - bridge_v2_name := BridgeName, - bridge_v2_conf := NewBridgeV2RawConf - } = - split_and_validate_bridge_v1_config(BridgeV1Type, BridgeName, RawConf, PreviousRawConf), +split_bridge_v1_config_and_create_helper( + BridgeV1Type, BridgeName, RawConf, PreviousRawConf, PreCreateFun +) -> + try + #{ + connector_type := ConnectorType, + connector_name := NewConnectorName, + connector_conf := NewConnectorRawConf, + bridge_v2_type := BridgeType, + bridge_v2_name := BridgeName, + bridge_v2_conf := NewBridgeV2RawConf + } = split_and_validate_bridge_v1_config( + BridgeV1Type, + BridgeName, + RawConf, + PreviousRawConf + ), + + _ = PreCreateFun(), + + do_connector_and_bridge_create( + ConnectorType, + NewConnectorName, + NewConnectorRawConf, + BridgeType, + BridgeName, + NewBridgeV2RawConf, + RawConf + ) + catch + throw:Reason -> + {error, Reason} + end. + +do_connector_and_bridge_create( + ConnectorType, + NewConnectorName, + NewConnectorRawConf, + BridgeType, + BridgeName, + NewBridgeV2RawConf, + RawConf +) -> case emqx_connector:create(ConnectorType, NewConnectorName, NewConnectorRawConf) of {ok, _} -> case create(BridgeType, BridgeName, NewBridgeV2RawConf) of @@ -1335,15 +1372,20 @@ bridge_v1_create_dry_run(BridgeType, RawConfig0) -> RawConf = maps:without([<<"name">>], RawConfig0), TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), PreviousRawConf = undefined, - #{ - connector_type := _ConnectorType, - connector_name := _NewConnectorName, - connector_conf := ConnectorRawConf, - bridge_v2_type := BridgeV2Type, - bridge_v2_name := _BridgeName, - bridge_v2_conf := BridgeV2RawConf - } = split_and_validate_bridge_v1_config(BridgeType, TmpName, RawConf, PreviousRawConf), - create_dry_run_helper(BridgeV2Type, ConnectorRawConf, BridgeV2RawConf). + try + #{ + connector_type := _ConnectorType, + connector_name := _NewConnectorName, + connector_conf := ConnectorRawConf, + bridge_v2_type := BridgeV2Type, + bridge_v2_name := _BridgeName, + bridge_v2_conf := BridgeV2RawConf + } = split_and_validate_bridge_v1_config(BridgeType, TmpName, RawConf, PreviousRawConf), + create_dry_run_helper(BridgeV2Type, ConnectorRawConf, BridgeV2RawConf) + catch + throw:Reason -> + {error, Reason} + end. %% Only called by test cases (may create broken references) bridge_v1_remove(BridgeV1Type, BridgeName) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index 13e84f84e..e6fcca50a 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -117,7 +117,7 @@ param_path_id() -> #{ in => path, required => true, - example => <<"webhook:webhook_example">>, + example => <<"http:my_http_action">>, desc => ?DESC("desc_param_path_id") } )}. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index 6adbf3942..b68a4c387 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -21,7 +21,7 @@ -export([ upgrade_pre_ee/2, maybe_upgrade/1, - webhook_maybe_upgrade/1 + http_maybe_upgrade/1 ]). upgrade_pre_ee(undefined, _UpgradeFunc) -> @@ -40,10 +40,10 @@ maybe_upgrade(#{<<"connector">> := _} = Config0) -> maybe_upgrade(NewVersion) -> NewVersion. -webhook_maybe_upgrade(#{<<"direction">> := _} = Config0) -> +http_maybe_upgrade(#{<<"direction">> := _} = Config0) -> Config1 = maps:remove(<<"direction">>, Config0), Config1#{<<"resource_opts">> => default_resource_opts()}; -webhook_maybe_upgrade(NewVersion) -> +http_maybe_upgrade(NewVersion) -> NewVersion. binary_key({K, V}) -> diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index ff924ac8c..27b3a8f14 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -162,13 +162,14 @@ roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})} fields(bridges) -> [ - {webhook, + {http, mk( hoconsc:map(name, ref(emqx_bridge_http_schema, "config")), #{ + aliases => [webhook], desc => ?DESC("bridges_webhook"), required => false, - converter => fun webhook_bridge_converter/2 + converter => fun http_bridge_converter/2 } )}, {mqtt, @@ -243,7 +244,7 @@ status() -> node_name() -> {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. -webhook_bridge_converter(Conf0, _HoconOpts) -> +http_bridge_converter(Conf0, _HoconOpts) -> emqx_bridge_compatible_config:upgrade_pre_ee( - Conf0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1 + Conf0, fun emqx_bridge_compatible_config:http_maybe_upgrade/1 ). diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index bc8be5476..30107d0ce 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -30,14 +30,18 @@ init_per_suite(Config) -> [ emqx, emqx_conf, + emqx_connector, + emqx_bridge_http, emqx_bridge ], #{work_dir => ?config(priv_dir, Config)} ), + emqx_mgmt_api_test_util:init_suite(), [{apps, Apps} | Config]. end_per_suite(Config) -> Apps = ?config(apps, Config), + emqx_mgmt_api_test_util:end_suite(), ok = emqx_cth_suite:stop(Apps), ok. @@ -58,6 +62,7 @@ end_per_testcase(t_get_basic_usage_info_1, _Config) -> ok = emqx_bridge:remove(BridgeType, BridgeName) end, [ + %% Keep using the old bridge names to avoid breaking the tests {webhook, <<"basic_usage_info_webhook">>}, {webhook, <<"basic_usage_info_webhook_disabled">>}, {mqtt, <<"basic_usage_info_mqtt">>} @@ -88,7 +93,7 @@ t_get_basic_usage_info_1(_Config) -> #{ num_bridges => 3, count_by_type => #{ - webhook => 1, + http => 1, mqtt => 2 } }, @@ -119,40 +124,33 @@ setup_fake_telemetry_data() -> HTTPConfig = #{ url => <<"http://localhost:9901/messages/${topic}">>, enable => true, - local_topic => "emqx_webhook/#", + local_topic => "emqx_http/#", method => post, body => <<"${payload}">>, headers => #{}, request_timeout => "15s" }, - Conf = - #{ - <<"bridges">> => - #{ - <<"webhook">> => - #{ - <<"basic_usage_info_webhook">> => HTTPConfig, - <<"basic_usage_info_webhook_disabled">> => - HTTPConfig#{enable => false} - }, - <<"mqtt">> => - #{ - <<"basic_usage_info_mqtt">> => MQTTConfig1, - <<"basic_usage_info_mqtt_from_select">> => MQTTConfig2 - } - } - }, - ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf), - - ok = snabbkaffe:start_trace(), - Predicate = fun(#{?snk_kind := K}) -> K =:= emqx_bridge_loaded end, - NEvents = 3, - BackInTime = 0, - Timeout = 11_000, - {ok, Sub} = snabbkaffe_collector:subscribe(Predicate, NEvents, Timeout, BackInTime), - ok = emqx_bridge:load(), - {ok, _} = snabbkaffe_collector:receive_events(Sub), - ok = snabbkaffe:stop(), + %% Keep use the old bridge names to test the backward compatibility + {ok, _} = emqx_bridge_testlib:create_bridge_api( + <<"webhook">>, + <<"basic_usage_info_webhook">>, + HTTPConfig + ), + {ok, _} = emqx_bridge_testlib:create_bridge_api( + <<"webhook">>, + <<"basic_usage_info_webhook_disabled">>, + HTTPConfig#{enable => false} + ), + {ok, _} = emqx_bridge_testlib:create_bridge_api( + <<"mqtt">>, + <<"basic_usage_info_mqtt">>, + MQTTConfig1 + ), + {ok, _} = emqx_bridge_testlib:create_bridge_api( + <<"mqtt">>, + <<"basic_usage_info_mqtt_from_select">>, + MQTTConfig2 + ), ok. t_update_ssl_conf(Config) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index ccc944572..e88206ccd 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -73,13 +73,15 @@ -define(HTTP_BRIDGE(URL), ?HTTP_BRIDGE(URL, ?BRIDGE_NAME)). -define(APPSPECS, [ - emqx_conf, emqx, + emqx_conf, emqx_auth, emqx_auth_mnesia, emqx_management, - {emqx_rule_engine, "rule_engine { rules {} }"}, - {emqx_bridge, "bridges {}"} + emqx_connector, + emqx_bridge_http, + {emqx_bridge, "actions {}\n bridges {}"}, + {emqx_rule_engine, "rule_engine { rules {} }"} ]). -define(APPSPEC_DASHBOARD, @@ -108,7 +110,7 @@ groups() -> ]. suite() -> - [{timetrap, {seconds, 60}}]. + [{timetrap, {seconds, 120}}]. init_per_suite(Config) -> Config. @@ -117,10 +119,10 @@ end_per_suite(_Config) -> ok. init_per_group(cluster = Name, Config) -> - Nodes = [NodePrimary | _] = mk_cluster(Config), + Nodes = [NodePrimary | _] = mk_cluster(Name, Config), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); init_per_group(cluster_later_join = Name, Config) -> - Nodes = [NodePrimary | _] = mk_cluster(Config, #{join_to => undefined}), + Nodes = [NodePrimary | _] = mk_cluster(Name, Config, #{join_to => undefined}), init_api([{group, Name}, {cluster_nodes, Nodes}, {node, NodePrimary} | Config]); init_per_group(_Name, Config) -> WorkDir = emqx_cth_suite:work_dir(Config), @@ -132,10 +134,10 @@ init_api(Config) -> {ok, App} = erpc:call(APINode, emqx_common_test_http, create_default_app, []), [{api, App} | Config]. -mk_cluster(Config) -> - mk_cluster(Config, #{}). +mk_cluster(Name, Config) -> + mk_cluster(Name, Config, #{}). -mk_cluster(Config, Opts) -> +mk_cluster(Name, Config, Opts) -> Node1Apps = ?APPSPECS ++ [?APPSPEC_DASHBOARD], Node2Apps = ?APPSPECS, emqx_cth_cluster:start( @@ -143,7 +145,7 @@ mk_cluster(Config, Opts) -> {emqx_bridge_api_SUITE1, Opts#{role => core, apps => Node1Apps}}, {emqx_bridge_api_SUITE2, Opts#{role => core, apps => Node2Apps}} ], - #{work_dir => emqx_cth_suite:work_dir(Config)} + #{work_dir => emqx_cth_suite:work_dir(Name, Config)} ). end_per_group(Group, Config) when @@ -159,7 +161,7 @@ init_per_testcase(t_broken_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), meck:expect(emqx_bpapi, supported_version, 1, -1), meck:expect(emqx_bpapi, supported_version, 2, -1), - init_per_testcase(commong, Config); + init_per_testcase(common, Config); init_per_testcase(t_old_bpapi_vsn, Config) -> meck:new(emqx_bpapi, [passthrough]), meck:expect(emqx_bpapi, supported_version, 1, 1), @@ -185,6 +187,18 @@ end_per_testcase(_, Config) -> ok. clear_resources() -> + lists:foreach( + fun(#{type := Type, name := Name}) -> + ok = emqx_bridge_v2:remove(Type, Name) + end, + emqx_bridge_v2:list() + ), + lists:foreach( + fun(#{type := Type, name := Name}) -> + ok = emqx_connector:remove(Type, Name) + end, + emqx_connector:list() + ), lists:foreach( fun(#{type := Type, name := Name}) -> ok = emqx_bridge:remove(Type, Name) @@ -407,10 +421,7 @@ t_http_crud_apis(Config) -> Config ), ?assertMatch( - #{ - <<"reason">> := <<"unknown_fields">>, - <<"unknown">> := <<"curl">> - }, + #{<<"reason">> := <<"required_field">>}, json(maps:get(<<"message">>, PutFail2)) ), {ok, 400, _} = request_json( @@ -419,12 +430,16 @@ t_http_crud_apis(Config) -> ?HTTP_BRIDGE(<<"localhost:1234/foo">>, Name), Config ), - {ok, 400, _} = request_json( + {ok, 400, PutFail3} = request_json( put, uri(["bridges", BridgeID]), ?HTTP_BRIDGE(<<"htpp://localhost:12341234/foo">>, Name), Config ), + ?assertMatch( + #{<<"kind">> := <<"validation_error">>}, + json(maps:get(<<"message">>, PutFail3)) + ), %% delete the bridge {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), @@ -463,7 +478,7 @@ t_http_crud_apis(Config) -> ), %% Create non working bridge - BrokenURL = ?URL(Port + 1, "/foo"), + BrokenURL = ?URL(Port + 1, "foo"), {ok, 201, BrokenBridge} = request( post, uri(["bridges"]), @@ -471,6 +486,7 @@ t_http_crud_apis(Config) -> fun json/1, Config ), + ?assertMatch( #{ <<"type">> := ?BRIDGE_TYPE_HTTP, @@ -1307,7 +1323,9 @@ t_cluster_later_join_metrics(Config) -> Name = ?BRIDGE_NAME, BridgeParams = ?HTTP_BRIDGE(URL1, Name), BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), + ?check_trace( + #{timetrap => 15_000}, begin %% Create a bridge on only one of the nodes. ?assertMatch({ok, 201, _}, request_json(post, uri(["bridges"]), BridgeParams, Config)), @@ -1319,8 +1337,26 @@ t_cluster_later_join_metrics(Config) -> }}, request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) ), + + ct:print("node joining cluster"), %% Now join the other node join with the api node. ok = erpc:call(OtherNode, ekka, join, [PrimaryNode]), + %% Hack / workaround for the fact that `emqx_machine_boot' doesn't restart the + %% applications, in particular `emqx_conf' doesn't restart and synchronize the + %% transaction id. It's also unclear at the moment why the equivalent test in + %% `emqx_bridge_v2_api_SUITE' doesn't need this hack. + ok = erpc:call(OtherNode, application, stop, [emqx_conf]), + ok = erpc:call(OtherNode, application, start, [emqx_conf]), + ct:print("node joined cluster"), + + %% assert: wait for the bridge to be ready on the other node. + {_, {ok, _}} = + ?wait_async_action( + {emqx_cluster_rpc, OtherNode} ! wake_up, + #{?snk_kind := cluster_rpc_caught_up, ?snk_meta := #{node := OtherNode}}, + 10_000 + ), + %% Check metrics; shouldn't crash even if the bridge is not %% ready on the node that just joined the cluster. ?assertMatch( @@ -1373,17 +1409,16 @@ t_create_with_bad_name(Config) -> validate_resource_request_ttl(single, Timeout, Name) -> SentData = #{payload => <<"Hello EMQX">>, timestamp => 1668602148000}, - BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), - ResId = emqx_bridge_resource:resource_id(<<"webhook">>, Name), + _BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_HTTP, Name), ?check_trace( begin {ok, Res} = ?wait_async_action( - emqx_bridge:send_message(BridgeID, SentData), + do_send_message(?BRIDGE_TYPE_HTTP, Name, SentData), #{?snk_kind := async_query}, 1000 ), - ?assertMatch({ok, #{id := ResId, query_opts := #{timeout := Timeout}}}, Res) + ?assertMatch({ok, #{id := _ResId, query_opts := #{timeout := Timeout}}}, Res) end, fun(Trace0) -> Trace = ?of_kind(async_query, Trace0), @@ -1394,6 +1429,10 @@ validate_resource_request_ttl(single, Timeout, Name) -> validate_resource_request_ttl(_Cluster, _Timeout, _Name) -> ignore. +do_send_message(BridgeV1Type, Name, Message) -> + Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(BridgeV1Type), + emqx_bridge_v2:send_message(Type, Name, Message, #{}). + %% request(Method, URL, Config) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl index 540c18878..b267e9bf7 100644 --- a/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_compatible_config_tests.erl @@ -21,7 +21,7 @@ empty_config_test() -> Conf1 = #{<<"bridges">> => #{}}, Conf2 = #{<<"bridges">> => #{<<"webhook">> => #{}}}, ?assertEqual(Conf1, check(Conf1)), - ?assertEqual(Conf2, check(Conf2)), + ?assertEqual(#{<<"bridges">> => #{<<"http">> => #{}}}, check(Conf2)), ok. %% ensure webhook config can be checked @@ -33,7 +33,7 @@ webhook_config_test() -> ?assertMatch( #{ <<"bridges">> := #{ - <<"webhook">> := #{ + <<"http">> := #{ <<"the_name">> := #{ <<"method">> := get, @@ -48,7 +48,7 @@ webhook_config_test() -> ?assertMatch( #{ <<"bridges">> := #{ - <<"webhook">> := #{ + <<"http">> := #{ <<"the_name">> := #{ <<"method">> := get, @@ -61,7 +61,7 @@ webhook_config_test() -> ), #{ <<"bridges">> := #{ - <<"webhook">> := #{ + <<"http">> := #{ <<"the_name">> := #{ <<"method">> := get, @@ -84,7 +84,7 @@ up(#{<<"mqtt">> := MqttBridges0} = Bridges) -> Bridges#{<<"mqtt">> := MqttBridges}; up(#{<<"webhook">> := WebhookBridges0} = Bridges) -> WebhookBridges = emqx_bridge_compatible_config:upgrade_pre_ee( - WebhookBridges0, fun emqx_bridge_compatible_config:webhook_maybe_upgrade/1 + WebhookBridges0, fun emqx_bridge_compatible_config:http_maybe_upgrade/1 ), Bridges#{<<"webhook">> := WebhookBridges}. @@ -126,7 +126,7 @@ check(Conf) when is_map(Conf) -> %% erlfmt-ignore %% this is config generated from v5.0.11 webhook_v5011_hocon() -> -""" +" bridges{ webhook { the_name{ @@ -143,7 +143,7 @@ bridges{ } } } -""". +". full_webhook_v5011_hocon() -> "" @@ -215,7 +215,7 @@ full_webhook_v5019_hocon() -> %% erlfmt-ignore %% this is a generated from v5.0.11 mqtt_v5011_hocon() -> -""" +" bridges { mqtt { bridge_one { @@ -257,12 +257,12 @@ bridges { } } } -""". +". %% erlfmt-ignore %% a more complete version mqtt_v5011_full_hocon() -> -""" +" bridges { mqtt { bridge_one { @@ -330,4 +330,4 @@ bridges { } } } -""". +". diff --git a/apps/emqx_bridge/test/emqx_bridge_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_testlib.erl index f486e5d64..118802551 100644 --- a/apps/emqx_bridge/test/emqx_bridge_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_testlib.erl @@ -92,7 +92,7 @@ end_per_testcase(_Testcase, Config) -> delete_all_bridges() -> lists:foreach( fun(#{name := Name, type := Type}) -> - emqx_bridge:remove(Type, Name) + ok = emqx_bridge:remove(Type, Name) end, emqx_bridge:list() ). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 8758c325d..83a857b47 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -185,7 +185,7 @@ mk_cluster(Name, Config, Opts) -> {emqx_bridge_v2_api_SUITE_1, Opts#{role => core, apps => Node1Apps}}, {emqx_bridge_v2_api_SUITE_2, Opts#{role => core, apps => Node2Apps}} ], - #{work_dir => filename:join(?config(priv_dir, Config), Name)} + #{work_dir => emqx_cth_suite:work_dir(Name, Config)} ). end_per_group(Group, Config) when diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl index 1ed0eb31b..a6b92caaa 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_testlib.erl @@ -146,6 +146,35 @@ create_bridge(Config, Overrides) -> ct:pal("creating bridge with config: ~p", [BridgeConfig]), emqx_bridge_v2:create(BridgeType, BridgeName, BridgeConfig). +maybe_json_decode(X) -> + case emqx_utils_json:safe_decode(X, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> X + end. + +request(Method, Path, Params) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + case emqx_mgmt_api_test_util:request_api(Method, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + Body = maybe_json_decode(Body0), + {ok, {Status, Headers, Body}}; + {error, {Status, Headers, Body0}} -> + Body = + case emqx_utils_json:safe_decode(Body0, [return_maps]) of + {ok, Decoded0 = #{<<"message">> := Msg0}} -> + Msg = maybe_json_decode(Msg0), + Decoded0#{<<"message">> := Msg}; + {ok, Decoded0} -> + Decoded0; + {error, _} -> + Body0 + end, + {error, {Status, Headers, Body}}; + Error -> + Error + end. + list_bridges_api() -> Params = [], Path = emqx_mgmt_api_test_util:api_path(["actions"]), @@ -209,6 +238,50 @@ create_bridge_api(Config, Overrides) -> ct:pal("bridge create result: ~p", [Res]), Res. +create_connector_api(Config) -> + create_connector_api(Config, _Overrides = #{}). + +create_connector_api(Config, Overrides) -> + ConnectorConfig0 = ?config(connector_config, Config), + ConnectorName = ?config(connector_name, Config), + ConnectorType = ?config(connector_type, Config), + Method = post, + Path = emqx_mgmt_api_test_util:api_path(["connectors"]), + ConnectorConfig = emqx_utils_maps:deep_merge(ConnectorConfig0, Overrides), + Params = ConnectorConfig#{<<"type">> => ConnectorType, <<"name">> => ConnectorName}, + ct:pal("creating connector (http):\n ~p", [Params]), + Res = request(Method, Path, Params), + ct:pal("connector create (http) result:\n ~p", [Res]), + Res. + +create_action_api(Config) -> + create_action_api(Config, _Overrides = #{}). + +create_action_api(Config, Overrides) -> + ActionName = ?config(action_name, Config), + ActionType = ?config(action_type, Config), + ActionConfig0 = ?config(action_config, Config), + ActionConfig = emqx_utils_maps:deep_merge(ActionConfig0, Overrides), + Params = ActionConfig#{<<"type">> => ActionType, <<"name">> => ActionName}, + Method = post, + Path = emqx_mgmt_api_test_util:api_path(["actions"]), + ct:pal("creating action (http):\n ~p", [Params]), + Res = request(Method, Path, Params), + ct:pal("action create (http) result:\n ~p", [Res]), + Res. + +get_action_api(Config) -> + ActionName = ?config(action_name, Config), + ActionType = ?config(action_type, Config), + ActionId = emqx_bridge_resource:bridge_id(ActionType, ActionName), + Params = [], + Method = get, + Path = emqx_mgmt_api_test_util:api_path(["actions", ActionId]), + ct:pal("getting action (http)"), + Res = request(Method, Path, Params), + ct:pal("get action (http) result:\n ~p", [Res]), + Res. + update_bridge_api(Config) -> update_bridge_api(Config, _Overrides = #{}). diff --git a/apps/emqx_bridge_azure_event_hub/rebar.config b/apps/emqx_bridge_azure_event_hub/rebar.config index efe337029..90be538b3 100644 --- a/apps/emqx_bridge_azure_event_hub/rebar.config +++ b/apps/emqx_bridge_azure_event_hub/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}} - , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}} + , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} , {snappyer, "1.2.9"} , {emqx_connector, {path, "../../apps/emqx_connector"}} diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src index f1c097d29..12d0890c3 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_azure_event_hub, [ {description, "EMQX Enterprise Azure Event Hub Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl index 92d268d20..1b135d0f7 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_tests.erl @@ -12,7 +12,7 @@ %% erlfmt-ignore aeh_producer_hocon() -> -""" +" bridges.azure_event_hub_producer.my_producer { enable = true authentication { @@ -62,7 +62,7 @@ bridges.azure_event_hub_producer.my_producer { server_name_indication = auto } } -""". +". %%=========================================================================== %% Helper functions diff --git a/apps/emqx_bridge_confluent/rebar.config b/apps/emqx_bridge_confluent/rebar.config index 38173e74c..0c0c2eece 100644 --- a/apps/emqx_bridge_confluent/rebar.config +++ b/apps/emqx_bridge_confluent/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}} - , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}} + , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} , {snappyer, "1.2.9"} , {emqx_connector, {path, "../../apps/emqx_connector"}} diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl index 16e6e11fe..a7efebf89 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_tests.erl @@ -12,7 +12,7 @@ %% erlfmt-ignore confluent_producer_action_hocon() -> -""" +" actions.confluent_producer.my_producer { enable = true connector = my_connector @@ -40,7 +40,7 @@ actions.confluent_producer.my_producer { } local_topic = \"t/confluent\" } -""". +". confluent_producer_connector_hocon() -> "" diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 6e2c93d20..59a02c190 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl index 6b64a02e9..44b2d022a 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl @@ -237,7 +237,10 @@ handle_continue(?patch_subscription, State0) -> ), {noreply, State0}; error -> - %% retry + %% retry; add a random delay for the case where multiple workers step on each + %% other's toes before retrying. + RandomMS = rand:uniform(500), + timer:sleep(RandomMS), {noreply, State0, {continue, ?patch_subscription}} end. diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 7e90ab48a..86f81277c 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -196,7 +196,7 @@ consumer_config(TestCase, Config) -> " connect_timeout = \"5s\"\n" " service_account_json = ~s\n" " consumer {\n" - " ack_deadline = \"60s\"\n" + " ack_deadline = \"10s\"\n" " ack_retry_interval = \"1s\"\n" " pull_max_messages = 10\n" " consumer_workers_per_topic = 1\n" @@ -520,7 +520,14 @@ wait_acked(Opts) -> {ok, _} -> ok; {timeout, Evts} -> - ct:pal("timed out waiting for acks; received:\n ~p", [Evts]) + %% Fixme: apparently, snabbkaffe may timeout but still return the expected + %% events here. + case length(Evts) >= N of + true -> + ok; + false -> + ct:pal("timed out waiting for acks;\n expected: ~b\n received:\n ~p", [N, Evts]) + end end, ok. @@ -658,25 +665,24 @@ setup_and_start_listeners(Node, NodeOpts) -> end ). +dedup([]) -> + []; +dedup([X]) -> + [X]; +dedup([X | Rest]) -> + [X | dedup(X, Rest)]. + +dedup(X, [X | Rest]) -> + dedup(X, Rest); +dedup(_X, [Y | Rest]) -> + [Y | dedup(Y, Rest)]; +dedup(_X, []) -> + []. + %%------------------------------------------------------------------------------ %% Trace properties %%------------------------------------------------------------------------------ -prop_pulled_only_once() -> - {"all pulled message ids are unique", fun ?MODULE:prop_pulled_only_once/1}. -prop_pulled_only_once(Trace) -> - PulledIds = - [ - MsgId - || #{messages := Msgs} <- ?of_kind(gcp_pubsub_consumer_worker_decoded_messages, Trace), - #{<<"message">> := #{<<"messageId">> := MsgId}} <- Msgs - ], - NumPulled = length(PulledIds), - UniquePulledIds = sets:from_list(PulledIds, [{version, 2}]), - UniqueNumPulled = sets:size(UniquePulledIds), - ?assertEqual(UniqueNumPulled, NumPulled, #{pulled_ids => PulledIds}), - ok. - prop_handled_only_once() -> {"all pulled message are processed only once", fun ?MODULE:prop_handled_only_once/1}. prop_handled_only_once(Trace) -> @@ -1052,7 +1058,6 @@ t_consume_ok(Config) -> end, [ prop_all_pulled_are_acked(), - prop_pulled_only_once(), prop_handled_only_once(), prop_acked_ids_eventually_forgotten() ] @@ -1125,7 +1130,6 @@ t_bridge_rule_action_source(Config) -> #{payload => Payload0} end, [ - prop_pulled_only_once(), prop_handled_only_once() ] ), @@ -1243,7 +1247,6 @@ t_multiple_topic_mappings(Config) -> end, [ prop_all_pulled_are_acked(), - prop_pulled_only_once(), prop_handled_only_once() ] ), @@ -1276,7 +1279,7 @@ t_multiple_pull_workers(Config) -> }, <<"resource_opts">> => #{ %% reduce flakiness - <<"request_ttl">> => <<"11s">> + <<"request_ttl">> => <<"20s">> } } ), @@ -1304,7 +1307,6 @@ t_multiple_pull_workers(Config) -> end, [ prop_all_pulled_are_acked(), - prop_pulled_only_once(), prop_handled_only_once(), {"message is processed only once", fun(Trace) -> ?assertMatch({timeout, _}, receive_published(#{timeout => 5_000})), @@ -1543,7 +1545,7 @@ t_async_worker_death_mid_pull(Config) -> fun(AsyncWorkerPid) -> Ref = monitor(process, AsyncWorkerPid), ct:pal("killing pid ~p", [AsyncWorkerPid]), - sys:terminate(AsyncWorkerPid, die, Timeout), + exit(AsyncWorkerPid, kill), receive {'DOWN', Ref, process, AsyncWorkerPid, _} -> ct:pal("killed pid ~p", [AsyncWorkerPid]), @@ -1605,18 +1607,19 @@ t_async_worker_death_mid_pull(Config) -> ], Trace ), + SubTraceEvts = ?projection(?snk_kind, SubTrace), ?assertMatch( [ - #{?snk_kind := gcp_pubsub_consumer_worker_handled_async_worker_down}, - #{?snk_kind := gcp_pubsub_consumer_worker_reply_delegator} + gcp_pubsub_consumer_worker_handled_async_worker_down, + gcp_pubsub_consumer_worker_reply_delegator | _ ], - SubTrace, + dedup(SubTraceEvts), #{sub_trace => projection_optional_span(SubTrace)} ), ?assertMatch( - #{?snk_kind := gcp_pubsub_consumer_worker_pull_response_received}, - lists:last(SubTrace) + gcp_pubsub_consumer_worker_pull_response_received, + lists:last(SubTraceEvts) ), ok end @@ -1948,7 +1951,6 @@ t_connection_down_during_ack(Config) -> end, [ prop_all_pulled_are_acked(), - prop_pulled_only_once(), prop_handled_only_once(), {"message is processed only once", fun(Trace) -> ?assertMatch({timeout, _}, receive_published(#{timeout => 5_000})), @@ -1973,7 +1975,15 @@ t_connection_down_during_ack_redeliver(Config) -> ?wait_async_action( create_bridge( Config, - #{<<"consumer">> => #{<<"ack_deadline">> => <<"10s">>}} + #{ + <<"consumer">> => #{ + <<"ack_deadline">> => <<"12s">>, + <<"ack_retry_interval">> => <<"1s">> + }, + <<"resource_opts">> => #{ + <<"request_ttl">> => <<"11s">> + } + } ), #{?snk_kind := "gcp_pubsub_consumer_worker_subscription_ready"}, 10_000 diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl index 885754470..de7467f62 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl @@ -12,7 +12,7 @@ %% erlfmt-ignore gcp_pubsub_producer_hocon() -> -""" +" bridges.gcp_pubsub.my_producer { attributes_template = [ {key = \"${payload.key}\", value = fixed_value} @@ -54,7 +54,7 @@ bridges.gcp_pubsub.my_producer { type = service_account } } -""". +". %%=========================================================================== %% Helper functions diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index a8a938a0b..c28c3ed92 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_greptimedb, [ {description, "EMQX GreptimeDB Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index 87d7e57a6..9cd71323e 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -3,7 +3,7 @@ {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]}, - {env, []}, + {env, [{emqx_action_info_modules, [emqx_bridge_http_action_info]}]}, {modules, []}, {links, []} ]}. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl new file mode 100644 index 000000000..457d8ff4b --- /dev/null +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_action_info.erl @@ -0,0 +1,102 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_http_action_info). + +-behaviour(emqx_action_info). + +-export([ + bridge_v1_type_name/0, + action_type_name/0, + connector_type_name/0, + schema_module/0, + connector_action_config_to_bridge_v1_config/2, + bridge_v1_config_to_action_config/2, + bridge_v1_config_to_connector_config/1 +]). + +-define(REMOVED_KEYS, [<<"direction">>]). +-define(ACTION_KEYS, [<<"local_topic">>, <<"resource_opts">>]). +-define(PARAMETER_KEYS, [<<"body">>, <<"max_retries">>, <<"method">>, <<"request_timeout">>]). + +bridge_v1_type_name() -> webhook. + +action_type_name() -> http. + +connector_type_name() -> http. + +schema_module() -> emqx_bridge_http_schema. + +connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> + BridgeV1Config1 = maps:remove(<<"connector">>, ActionConfig), + %% Move parameters to the top level + ParametersMap1 = maps:get(<<"parameters">>, BridgeV1Config1, #{}), + ParametersMap2 = maps:without([<<"path">>, <<"headers">>], ParametersMap1), + BridgeV1Config2 = maps:remove(<<"parameters">>, BridgeV1Config1), + BridgeV1Config3 = emqx_utils_maps:deep_merge(BridgeV1Config2, ParametersMap2), + BridgeV1Config4 = emqx_utils_maps:deep_merge(ConnectorConfig, BridgeV1Config3), + + Url = maps:get(<<"url">>, ConnectorConfig), + Path = maps:get(<<"path">>, ParametersMap1, <<>>), + + Headers1 = maps:get(<<"headers">>, ConnectorConfig, #{}), + Headers2 = maps:get(<<"headers">>, ParametersMap1, #{}), + + Url1 = + case Path of + <<>> -> Url; + _ -> iolist_to_binary(emqx_bridge_http_connector:join_paths(Url, Path)) + end, + + BridgeV1Config4#{ + <<"headers">> => maps:merge(Headers1, Headers2), + <<"url">> => Url1 + }. + +bridge_v1_config_to_connector_config(BridgeV1Conf) -> + %% To statisfy the emqx_bridge_api_SUITE:t_http_crud_apis/1 + ok = validate_webhook_url(maps:get(<<"url">>, BridgeV1Conf, undefined)), + maps:without(?REMOVED_KEYS ++ ?ACTION_KEYS ++ ?PARAMETER_KEYS, BridgeV1Conf). + +bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) -> + Parameters = maps:with(?PARAMETER_KEYS, BridgeV1Conf), + Parameters1 = Parameters#{<<"path">> => <<>>, <<"headers">> => #{}}, + CommonKeys = [<<"enable">>, <<"description">>], + ActionConfig = maps:with(?ACTION_KEYS ++ CommonKeys, BridgeV1Conf), + ActionConfig#{<<"parameters">> => Parameters1, <<"connector">> => ConnectorName}. + +%%-------------------------------------------------------------------- +%% helpers + +validate_webhook_url(undefined) -> + throw(#{ + kind => validation_error, + reason => required_field, + required_field => <<"url">> + }); +validate_webhook_url(Url) -> + {BaseUrl, _Path} = emqx_connector_resource:parse_url(Url), + case emqx_http_lib:uri_parse(BaseUrl) of + {ok, _} -> + ok; + {error, Reason} -> + throw(#{ + kind => validation_error, + reason => invalid_url, + url => Url, + error => emqx_utils:readable_error_msg(Reason) + }) + end. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 5a5e790e5..5ecfa76d1 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -31,9 +31,14 @@ on_query/3, on_query_async/4, on_get_status/2, - reply_delegator/3 + on_add_channel/4, + on_remove_channel/3, + on_get_channels/1, + on_get_channel_status/3 ]). +-export([reply_delegator/3]). + -export([ roots/0, fields/1, @@ -41,7 +46,7 @@ namespace/0 ]). -%% for other webhook-like connectors. +%% for other http-like connectors. -export([redact_request/1]). -export([validate_method/1, join_paths/2]). @@ -251,6 +256,21 @@ start_pool(PoolName, PoolOpts) -> Error end. +on_add_channel( + _InstId, + OldState, + ActionId, + ActionConfig +) -> + InstalledActions = maps:get(installed_actions, OldState, #{}), + {ok, ActionState} = do_create_http_action(ActionConfig), + NewInstalledActions = maps:put(ActionId, ActionState, InstalledActions), + NewState = maps:put(installed_actions, NewInstalledActions, OldState), + {ok, NewState}. + +do_create_http_action(_ActionConfig = #{parameters := Params}) -> + {ok, preprocess_request(Params)}. + on_stop(InstId, _State) -> ?SLOG(info, #{ msg => "stopping_http_connector", @@ -260,6 +280,16 @@ on_stop(InstId, _State) -> ?tp(emqx_connector_http_stopped, #{instance_id => InstId}), Res. +on_remove_channel( + _InstId, + OldState = #{installed_actions := InstalledActions}, + ActionId +) -> + NewInstalledActions = maps:remove(ActionId, InstalledActions), + NewState = maps:put(installed_actions, NewInstalledActions, OldState), + {ok, NewState}. + +%% BridgeV1 entrypoint on_query(InstId, {send_message, Msg}, State) -> case maps:get(request, State, undefined) of undefined -> @@ -282,6 +312,36 @@ on_query(InstId, {send_message, Msg}, State) -> State ) end; +%% BridgeV2 entrypoint +on_query( + InstId, + {ActionId, Msg}, + State = #{installed_actions := InstalledActions} +) when is_binary(ActionId) -> + case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of + {undefined, _} -> + ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}), + {error, arg_request_not_found}; + {_, undefined} -> + ?SLOG(error, #{msg => "action_not_found", connector => InstId, action_id => ActionId}), + {error, action_not_found}; + {Request, ActionState} -> + #{ + method := Method, + path := Path, + body := Body, + headers := Headers, + request_timeout := Timeout + } = process_request_and_action(Request, ActionState, Msg), + %% bridge buffer worker has retry, do not let ehttpc retry + Retry = 2, + ClientId = maps:get(clientid, Msg, undefined), + on_query( + InstId, + {ClientId, Method, {Path, Headers, Body}, Timeout, Retry}, + State + ) + end; on_query(InstId, {Method, Request}, State) -> %% TODO: Get retry from State on_query(InstId, {undefined, Method, Request, 5000, _Retry = 2}, State); @@ -343,6 +403,7 @@ on_query( Result end. +%% BridgeV1 entrypoint on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> case maps:get(request, State, undefined) of undefined -> @@ -364,6 +425,36 @@ on_query_async(InstId, {send_message, Msg}, ReplyFunAndArgs, State) -> State ) end; +%% BridgeV2 entrypoint +on_query_async( + InstId, + {ActionId, Msg}, + ReplyFunAndArgs, + State = #{installed_actions := InstalledActions} +) when is_binary(ActionId) -> + case {maps:get(request, State, undefined), maps:get(ActionId, InstalledActions, undefined)} of + {undefined, _} -> + ?SLOG(error, #{msg => "arg_request_not_found", connector => InstId}), + {error, arg_request_not_found}; + {_, undefined} -> + ?SLOG(error, #{msg => "action_not_found", connector => InstId, action_id => ActionId}), + {error, action_not_found}; + {Request, ActionState} -> + #{ + method := Method, + path := Path, + body := Body, + headers := Headers, + request_timeout := Timeout + } = process_request_and_action(Request, ActionState, Msg), + ClientId = maps:get(clientid, Msg, undefined), + on_query_async( + InstId, + {ClientId, Method, {Path, Headers, Body}, Timeout}, + ReplyFunAndArgs, + State + ) + end; on_query_async( InstId, {KeyOrNum, Method, Request, Timeout}, @@ -411,6 +502,9 @@ resolve_pool_worker(#{pool_name := PoolName} = State, Key) -> ehttpc_pool:pick_worker(PoolName, Key) end. +on_get_channels(ResId) -> + emqx_bridge_v2:get_channels_for_connector(ResId). + on_get_status(_InstId, #{pool_name := PoolName, connect_timeout := Timeout} = State) -> case do_get_status(PoolName, Timeout) of ok -> @@ -456,6 +550,14 @@ do_get_status(PoolName, Timeout) -> {error, timeout} end. +on_get_channel_status( + InstId, + _ChannelId, + State +) -> + %% XXX: Reuse the connector status + on_get_status(InstId, State). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -466,10 +568,10 @@ preprocess_request(Req) when map_size(Req) == 0 -> preprocess_request( #{ method := Method, - path := Path, - headers := Headers + path := Path } = Req ) -> + Headers = maps:get(headers, Req, []), #{ method => parse_template(to_bin(Method)), path => parse_template(Path), @@ -529,6 +631,49 @@ maybe_parse_template(Key, Conf) -> parse_template(String) -> emqx_template:parse(String). +process_request_and_action(Request, ActionState, Msg) -> + MethodTemplate = maps:get(method, ActionState), + Method = make_method(render_template_string(MethodTemplate, Msg)), + BodyTemplate = maps:get(body, ActionState), + Body = render_request_body(BodyTemplate, Msg), + + PathPrefix = unicode:characters_to_list(render_template(maps:get(path, Request), Msg)), + PathSuffix = unicode:characters_to_list(render_template(maps:get(path, ActionState), Msg)), + + Path = + case PathSuffix of + "" -> PathPrefix; + _ -> join_paths(PathPrefix, PathSuffix) + end, + + HeadersTemplate1 = maps:get(headers, Request), + HeadersTemplate2 = maps:get(headers, ActionState), + Headers = merge_proplist( + render_headers(HeadersTemplate1, Msg), + render_headers(HeadersTemplate2, Msg) + ), + #{ + method => Method, + path => Path, + body => Body, + headers => Headers, + request_timeout => maps:get(request_timeout, ActionState) + }. + +merge_proplist(Proplist1, Proplist2) -> + lists:foldl( + fun({K, V}, Acc) -> + case lists:keyfind(K, 1, Acc) of + false -> + [{K, V} | Acc]; + {K, _} = {K, V1} -> + [{K, V1} | Acc] + end + end, + Proplist2, + Proplist1 + ). + process_request( #{ method := MethodTemplate, @@ -691,7 +836,7 @@ maybe_retry({error, Reason}, Context, ReplyFunAndArgs) -> true -> Context; false -> Context#{attempt := Attempt + 1} end, - ?tp(webhook_will_retry_async, #{}), + ?tp(http_will_retry_async, #{}), Worker = resolve_pool_worker(State, KeyOrNum), ok = ehttpc:request_async( Worker, diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index 2e3d882d5..6a9219c11 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -18,69 +18,162 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --import(hoconsc, [mk/2, enum/1, ref/2]). +-import(hoconsc, [mk/2, enum/1, ref/1, ref/2]). -export([roots/0, fields/1, namespace/0, desc/1]). +-export([ + bridge_v2_examples/1, + %%conn_bridge_examples/1, + connector_examples/1 +]). + %%====================================================================================== %% Hocon Schema Definitions -namespace() -> "bridge_webhook". +namespace() -> "bridge_http". roots() -> []. -fields("config") -> - basic_config() ++ request_config(); +%%-------------------------------------------------------------------- +%% v1 bridges http api +%% see: emqx_bridge_schema:get_response/0, put_request/0, post_request/0 fields("post") -> [ - type_field(), + old_type_field(), name_field() ] ++ fields("config"); fields("put") -> fields("config"); fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"); -fields("creation_opts") -> +%%--- v1 bridges config file +%% see: emqx_bridge_schema:fields(bridges) +fields("config") -> + basic_config() ++ request_config(); +%%-------------------------------------------------------------------- +%% v2: configuration +fields(action) -> + {http, + mk( + hoconsc:map(name, ref(?MODULE, "http_action")), + #{ + aliases => [webhook], + desc => <<"HTTP Action Config">>, + required => false + } + )}; +fields("http_action") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable_bridge"), default => true})}, + {connector, + mk(binary(), #{ + desc => ?DESC(emqx_connector_schema, "connector_field"), required => true + })}, + {description, emqx_schema:description_schema()}, + %% Note: there's an implicit convention in `emqx_bridge' that, + %% for egress bridges with this config, the published messages + %% will be forwarded to such bridges. + {local_topic, + mk( + binary(), + #{ + required => false, + desc => ?DESC("config_local_topic"), + importance => ?IMPORTANCE_HIDDEN + } + )}, + %% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions. + %% some fields are moved to connector, some fields are moved to actions and composed into the + %% `parameters` field. + {parameters, + mk(ref("parameters_opts"), #{ + required => true, + desc => ?DESC("config_parameters_opts") + })} + ] ++ http_resource_opts(); +fields("parameters_opts") -> + [ + {path, + mk( + binary(), + #{ + desc => ?DESC("config_path"), + required => false + } + )}, + method_field(), + headers_field(), + body_field(), + max_retries_field(), + request_timeout_field() + ]; +%% v2: api schema +%% The parameter equls to +%% `get_bridge_v2`, `post_bridge_v2`, `put_bridge_v2` from emqx_bridge_v2_schema:api_schema/1 +%% `get_connector`, `post_connector`, `put_connector` from emqx_connector_schema:api_schema/1 +fields("post_" ++ Type) -> + [type_field(), name_field() | fields("config_" ++ Type)]; +fields("put_" ++ Type) -> + fields("config_" ++ Type); +fields("get_" ++ Type) -> + emqx_bridge_schema:status_fields() ++ fields("post_" ++ Type); +fields("config_bridge_v2") -> + fields("http_action"); +fields("config_connector") -> + [ + {enable, + mk( + boolean(), + #{ + desc => <<"Enable or disable this connector">>, + default => true + } + )}, + {description, emqx_schema:description_schema()} + ] ++ connector_url_headers() ++ connector_opts(); +%%-------------------------------------------------------------------- +%% v1/v2 +fields("resource_opts") -> + UnsupportedOpts = [enable_batch, batch_size, batch_time], lists:filter( - fun({K, _V}) -> - not lists:member(K, unsupported_opts()) - end, + fun({K, _V}) -> not lists:member(K, UnsupportedOpts) end, emqx_resource_schema:fields("creation_opts") ). desc("config") -> ?DESC("desc_config"); -desc("creation_opts") -> +desc("resource_opts") -> ?DESC(emqx_resource_schema, "creation_opts"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for WebHook using `", string:to_upper(Method), "` method."]; +desc("config_connector") -> + ?DESC("desc_config"); +desc("http_action") -> + ?DESC("desc_config"); +desc("parameters_opts") -> + ?DESC("config_parameters_opts"); desc(_) -> undefined. +%%-------------------------------------------------------------------- +%% helpers for v1 only + basic_config() -> [ {enable, mk( boolean(), #{ - desc => ?DESC("config_enable"), + desc => ?DESC("config_enable_bridge"), default => true } - )} - ] ++ webhook_creation_opts() ++ - proplists:delete( - max_retries, emqx_bridge_http_connector:fields(config) - ). + )}, + {description, emqx_schema:description_schema()} + ] ++ http_resource_opts() ++ connector_opts(). request_config() -> [ - {url, - mk( - binary(), - #{ - required => true, - desc => ?DESC("config_url") - } - )}, + url_field(), {direction, mk( egress, @@ -98,81 +191,37 @@ request_config() -> required => false } )}, - {method, - mk( - method(), - #{ - default => post, - desc => ?DESC("config_method") - } - )}, - {headers, - mk( - map(), - #{ - default => #{ - <<"accept">> => <<"application/json">>, - <<"cache-control">> => <<"no-cache">>, - <<"connection">> => <<"keep-alive">>, - <<"content-type">> => <<"application/json">>, - <<"keep-alive">> => <<"timeout=5">> - }, - desc => ?DESC("config_headers") - } - )}, - {body, - mk( - binary(), - #{ - default => undefined, - desc => ?DESC("config_body") - } - )}, - {max_retries, - mk( - non_neg_integer(), - #{ - default => 2, - desc => ?DESC("config_max_retries") - } - )}, - {request_timeout, - mk( - emqx_schema:duration_ms(), - #{ - default => <<"15s">>, - deprecated => {since, "v5.0.26"}, - desc => ?DESC("config_request_timeout") - } - )} + method_field(), + headers_field(), + body_field(), + max_retries_field(), + request_timeout_field() ]. -webhook_creation_opts() -> - [ - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } - )} - ]. +%%-------------------------------------------------------------------- +%% helpers for v2 only -unsupported_opts() -> - [ - enable_batch, - batch_size, - batch_time - ]. +connector_url_headers() -> + [url_field(), headers_field()]. -%%====================================================================================== +%%-------------------------------------------------------------------- +%% common funcs + +%% `webhook` is kept for backward compatibility. +old_type_field() -> + {type, + mk( + enum([webhook, http]), + #{ + required => true, + desc => ?DESC("desc_type") + } + )}. type_field() -> {type, mk( - webhook, + http, #{ required => true, desc => ?DESC("desc_type") @@ -189,5 +238,189 @@ name_field() -> } )}. -method() -> - enum([post, put, get, delete]). +url_field() -> + {url, + mk( + binary(), + #{ + required => true, + desc => ?DESC("config_url") + } + )}. + +headers_field() -> + {headers, + mk( + map(), + #{ + default => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keep-alive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">> + }, + desc => ?DESC("config_headers") + } + )}. + +method_field() -> + {method, + mk( + enum([post, put, get, delete]), + #{ + default => post, + desc => ?DESC("config_method") + } + )}. + +body_field() -> + {body, + mk( + binary(), + #{ + default => undefined, + desc => ?DESC("config_body") + } + )}. + +max_retries_field() -> + {max_retries, + mk( + non_neg_integer(), + #{ + default => 2, + desc => ?DESC("config_max_retries") + } + )}. + +request_timeout_field() -> + {request_timeout, + mk( + emqx_schema:duration_ms(), + #{ + default => <<"15s">>, + deprecated => {since, "v5.0.26"}, + desc => ?DESC("config_request_timeout") + } + )}. + +http_resource_opts() -> + [ + {resource_opts, + mk( + ref(?MODULE, "resource_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ]. + +connector_opts() -> + mark_request_field_deperecated( + proplists:delete(max_retries, emqx_bridge_http_connector:fields(config)) + ). + +mark_request_field_deperecated(Fields) -> + lists:map( + fun({K, V}) -> + case K of + request -> + {K, V#{ + %% Note: if we want to deprecate a reference type, we have to change + %% it to a direct type first. + type => typerefl:map(), + deprecated => {since, "5.3.2"}, + desc => <<"This field is never used, so we deprecated it since 5.3.2.">> + }}; + _ -> + {K, V} + end + end, + Fields + ). + +%%-------------------------------------------------------------------- +%% Examples + +bridge_v2_examples(Method) -> + [ + #{ + <<"http">> => #{ + summary => <<"HTTP Action">>, + value => values({Method, bridge_v2}) + } + } + ]. + +connector_examples(Method) -> + [ + #{ + <<"http">> => #{ + summary => <<"HTTP Connector">>, + value => values({Method, connector}) + } + } + ]. + +values({get, Type}) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + }, + values({post, Type}) + ); +values({post, bridge_v2}) -> + maps:merge( + #{ + name => <<"my_http_action">>, + type => <<"http">> + }, + values({put, bridge_v2}) + ); +values({post, connector}) -> + maps:merge( + #{ + name => <<"my_http_connector">>, + type => <<"http">> + }, + values({put, connector}) + ); +values({put, bridge_v2}) -> + values(bridge_v2); +values({put, connector}) -> + values(connector); +values(bridge_v2) -> + #{ + enable => true, + connector => <<"my_http_connector">>, + parameters => #{ + path => <<"/room/${room_no}">>, + method => <<"post">>, + headers => #{}, + body => <<"${.}">> + }, + resource_opts => #{ + worker_pool_size => 16, + health_check_interval => <<"15s">>, + query_mode => <<"async">> + } + }; +values(connector) -> + #{ + enable => true, + url => <<"http://localhost:8080/api/v1">>, + headers => #{<<"content-type">> => <<"application/json">>}, + connect_timeout => <<"15s">>, + pool_type => <<"hash">>, + pool_size => 1, + enable_pipelining => 100 + }. diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl index d9fc595fe..3b7303300 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_SUITE.erl @@ -39,18 +39,33 @@ all() -> groups() -> []. -init_per_suite(_Config) -> - emqx_common_test_helpers:render_and_load_app_config(emqx_conf), - ok = emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_bridge, emqx_rule_engine]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - []. +init_per_suite(Config0) -> + Config = + case os:getenv("DEBUG_CASE") of + [_ | _] = DebugCase -> + CaseName = list_to_atom(DebugCase), + [{debug_case, CaseName} | Config0]; + _ -> + Config0 + end, + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_http, + emqx_bridge, + emqx_rule_engine + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + emqx_mgmt_api_test_util:init_suite(), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - ok = emqx_mgmt_api_test_util:end_suite([emqx_rule_engine, emqx_bridge, emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), - _ = application:stop(emqx_bridge), +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_cth_suite:stop(Apps), ok. suite() -> @@ -115,7 +130,8 @@ end_per_testcase(TestCase, _Config) when -> ok = emqx_bridge_http_connector_test_server:stop(), persistent_term:erase({?MODULE, times_called}), - emqx_bridge_testlib:delete_all_bridges(), + emqx_bridge_v2_testlib:delete_all_bridges(), + emqx_bridge_v2_testlib:delete_all_connectors(), emqx_common_test_helpers:call_janitor(), ok; end_per_testcase(_TestCase, Config) -> @@ -123,7 +139,8 @@ end_per_testcase(_TestCase, Config) -> undefined -> ok; Server -> stop_http_server(Server) end, - emqx_bridge_testlib:delete_all_bridges(), + emqx_bridge_v2_testlib:delete_all_bridges(), + emqx_bridge_v2_testlib:delete_all_connectors(), emqx_common_test_helpers:call_janitor(), ok. @@ -420,7 +437,7 @@ t_send_async_connection_timeout(Config) -> ), NumberOfMessagesToSend = 10, [ - emqx_bridge:send_message(BridgeID, #{<<"id">> => Id}) + do_send_message(#{<<"id">> => Id}) || Id <- lists:seq(1, NumberOfMessagesToSend) ], %% Make sure server receives all messages @@ -431,7 +448,7 @@ t_send_async_connection_timeout(Config) -> t_async_free_retries(Config) -> #{port := Port} = ?config(http_server, Config), - BridgeID = make_bridge(#{ + _BridgeID = make_bridge(#{ port => Port, pool_size => 1, query_mode => "sync", @@ -445,7 +462,7 @@ t_async_free_retries(Config) -> Fn = fun(Get, Error) -> ?assertMatch( {ok, 200, _, _}, - emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + do_send_message(#{<<"hello">> => <<"world">>}), #{error => Error} ), ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) @@ -456,7 +473,7 @@ t_async_free_retries(Config) -> t_async_common_retries(Config) -> #{port := Port} = ?config(http_server, Config), - BridgeID = make_bridge(#{ + _BridgeID = make_bridge(#{ port => Port, pool_size => 1, query_mode => "sync", @@ -471,7 +488,7 @@ t_async_common_retries(Config) -> FnSucceed = fun(Get, Error) -> ?assertMatch( {ok, 200, _, _}, - emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + do_send_message(#{<<"hello">> => <<"world">>}), #{error => Error, attempts => Get()} ), ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) @@ -479,7 +496,7 @@ t_async_common_retries(Config) -> FnFail = fun(Get, Error) -> ?assertMatch( Error, - emqx_bridge:send_message(BridgeID, #{<<"hello">> => <<"world">>}), + do_send_message(#{<<"hello">> => <<"world">>}), #{error => Error, attempts => Get()} ), ?assertEqual(ExpectedAttempts, Get(), #{error => Error}) @@ -559,7 +576,7 @@ t_path_not_found(Config) -> ok end, fun(Trace) -> - ?assertEqual([], ?of_kind(webhook_will_retry_async, Trace)), + ?assertEqual([], ?of_kind(http_will_retry_async, Trace)), ok end ), @@ -600,7 +617,7 @@ t_too_many_requests(Config) -> ok end, fun(Trace) -> - ?assertMatch([_ | _], ?of_kind(webhook_will_retry_async, Trace)), + ?assertMatch([_ | _], ?of_kind(http_will_retry_async, Trace)), ok end ), @@ -711,6 +728,11 @@ t_bridge_probes_header_atoms(Config) -> ok. %% helpers + +do_send_message(Message) -> + Type = emqx_bridge_v2:bridge_v1_type_to_bridge_v2_type(?BRIDGE_TYPE), + emqx_bridge_v2:send_message(Type, ?BRIDGE_NAME, Message, #{}). + do_t_async_retries(TestCase, TestContext, Error, Fn) -> #{error_attempts := ErrorAttempts} = TestContext, PTKey = {?MODULE, TestCase, attempts}, diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl index 4f5e2929c..f2de91123 100644 --- a/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_connector_tests.erl @@ -175,7 +175,7 @@ check_atom_key(Conf) when is_map(Conf) -> %% erlfmt-ignore webhook_config_hocon() -> -""" +" bridges.webhook.a { body = \"${.}\" connect_timeout = 15s @@ -209,4 +209,4 @@ bridges.webhook.a { } url = \"http://some.host:4000/api/echo\" } -""". +". diff --git a/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl new file mode 100644 index 000000000..38d1d5a68 --- /dev/null +++ b/apps/emqx_bridge_http/test/emqx_bridge_http_v2_SUITE.erl @@ -0,0 +1,140 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_http_v2_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_mgmt_api_test_util, [request/3]). +-import(emqx_common_test_helpers, [on_exit/1]). +-import(emqx_bridge_http_SUITE, [start_http_server/1, stop_http_server/1]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/asserts.hrl"). + +-define(BRIDGE_TYPE, <<"http">>). +-define(BRIDGE_NAME, atom_to_binary(?MODULE)). +-define(CONNECTOR_NAME, atom_to_binary(?MODULE)). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config0) -> + Config = + case os:getenv("DEBUG_CASE") of + [_ | _] = DebugCase -> + CaseName = list_to_atom(DebugCase), + [{debug_case, CaseName} | Config0]; + _ -> + Config0 + end, + Apps = emqx_cth_suite:start( + [ + emqx, + emqx_conf, + emqx_connector, + emqx_bridge_http, + emqx_bridge, + emqx_rule_engine + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + emqx_mgmt_api_test_util:init_suite(), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_cth_suite:stop(Apps), + ok. + +suite() -> + [{timetrap, {seconds, 60}}]. + +init_per_testcase(_TestCase, Config) -> + Server = start_http_server(#{response_delay_ms => 0}), + [{http_server, Server} | Config]. + +end_per_testcase(_TestCase, Config) -> + case ?config(http_server, Config) of + undefined -> ok; + Server -> stop_http_server(Server) + end, + emqx_bridge_v2_testlib:delete_all_bridges(), + emqx_bridge_v2_testlib:delete_all_connectors(), + emqx_common_test_helpers:call_janitor(), + ok. + +%%-------------------------------------------------------------------- +%% tests +%%-------------------------------------------------------------------- + +t_compose_connector_url_and_action_path(Config) -> + Path = <<"/foo/bar">>, + ConnectorCfg = make_connector_config(Config), + ActionCfg = make_action_config([{path, Path} | Config]), + CreateConfig = [ + {bridge_type, ?BRIDGE_TYPE}, + {bridge_name, ?BRIDGE_NAME}, + {bridge_config, ActionCfg}, + {connector_type, ?BRIDGE_TYPE}, + {connector_name, ?CONNECTOR_NAME}, + {connector_config, ConnectorCfg} + ], + {ok, _} = emqx_bridge_v2_testlib:create_bridge(CreateConfig), + + %% assert: the url returned v1 api is composed by the url of the connector and the + %% path of the action + #{port := Port} = ?config(http_server, Config), + ExpectedUrl = iolist_to_binary(io_lib:format("http://localhost:~p/foo/bar", [Port])), + {ok, {_, _, [Bridge]}} = emqx_bridge_testlib:list_bridges_api(), + ?assertMatch( + #{<<"url">> := ExpectedUrl}, + Bridge + ), + ok. + +%%-------------------------------------------------------------------- +%% helpers +%%-------------------------------------------------------------------- + +make_connector_config(Config) -> + #{port := Port} = ?config(http_server, Config), + #{ + <<"enable">> => true, + <<"url">> => iolist_to_binary(io_lib:format("http://localhost:~p", [Port])), + <<"headers">> => #{}, + <<"pool_type">> => <<"hash">>, + <<"pool_size">> => 1 + }. + +make_action_config(Config) -> + Path = ?config(path, Config), + #{ + <<"enable">> => true, + <<"connector">> => ?CONNECTOR_NAME, + <<"parameters">> => #{ + <<"path">> => Path, + <<"method">> => <<"post">>, + <<"headers">> => #{}, + <<"body">> => <<"${.}">> + } + }. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index c6236d97c..ef288368d 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kafka/rebar.config b/apps/emqx_bridge_kafka/rebar.config index 92e83fa04..b69ec1262 100644 --- a/apps/emqx_bridge_kafka/rebar.config +++ b/apps/emqx_bridge_kafka/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.8.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}} - , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}} + , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} , {snappyer, "1.2.9"} , {emqx_connector, {path, "../../apps/emqx_connector"}} diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index da8df2ddc..1d9d5c807 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_kafka, [ {description, "EMQX Enterprise Kafka Bridge"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, [emqx_bridge_kafka_consumer_sup]}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 28050d368..951fb5ef5 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -269,7 +269,11 @@ fields(Field) when Field == "put_connector"; Field == "post_connector" -> - emqx_connector_schema:api_fields(Field, ?CONNECTOR_TYPE, kafka_connector_config_fields()); + emqx_connector_schema:api_fields( + Field, + ?CONNECTOR_TYPE, + kafka_connector_config_fields() + ); fields("post_" ++ Type) -> [type_field(Type), name_field() | fields("config_" ++ Type)]; fields("put_" ++ Type) -> @@ -508,8 +512,7 @@ fields(consumer_opts) -> {value_encoding_mode, mk(enum([none, base64]), #{ default => none, desc => ?DESC(consumer_value_encoding_mode) - })}, - {resource_opts, mk(ref(resource_opts), #{default => #{}})} + })} ]; fields(consumer_topic_mapping) -> [ @@ -623,7 +626,7 @@ kafka_connector_config_fields() -> })}, {socket_opts, mk(ref(socket_opts), #{required => false, desc => ?DESC(socket_opts)})}, {ssl, mk(ref(ssl_client_opts), #{})} - ]. + ] ++ [resource_opts()]. producer_opts(ActionOrBridgeV1) -> [ @@ -631,9 +634,11 @@ producer_opts(ActionOrBridgeV1) -> %% for egress bridges with this config, the published messages %% will be forwarded to such bridges. {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})}, - parameters_field(ActionOrBridgeV1), - {resource_opts, mk(ref(resource_opts), #{default => #{}, desc => ?DESC(resource_opts)})} - ]. + parameters_field(ActionOrBridgeV1) + ] ++ [resource_opts() || ActionOrBridgeV1 =:= action]. + +resource_opts() -> + {resource_opts, mk(ref(resource_opts), #{default => #{}, desc => ?DESC(resource_opts)})}. %% Since e5.3.1, we want to rename the field 'kafka' to 'parameters' %% However we need to keep it backward compatible for generated schema json (version 0.1.0) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 702e4592b..bf8c76bee 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -81,11 +81,24 @@ on_start(InstId, Config) -> ClientId = InstId, emqx_resource:allocate_resource(InstId, ?kafka_client_id, ClientId), ok = ensure_client(ClientId, Hosts, ClientConfig), - %% Check if this is a dry run - {ok, #{ - client_id => ClientId, - installed_bridge_v2s => #{} - }}. + %% Note: we must return `{error, _}' here if the client cannot connect so that the + %% connector will immediately enter the `?status_disconnected' state, and then avoid + %% giving the impression that channels/actions may be added immediately and start + %% buffering, which won't happen if it's `?status_connecting'. That would lead to + %% data loss, since Kafka Producer uses wolff's internal buffering, which is started + %% only when its producers start. + case check_client_connectivity(ClientId) of + ok -> + {ok, #{ + client_id => ClientId, + installed_bridge_v2s => #{} + }}; + {error, {find_client, Reason}} -> + %% Race condition? Crash? We just checked it with `ensure_client'... + {error, Reason}; + {error, {connectivity, Reason}} -> + {error, Reason} + end. on_add_channel( InstId, @@ -478,14 +491,18 @@ on_get_status( _InstId, #{client_id := ClientId} = State ) -> - case wolff_client_sup:find_client(ClientId) of - {ok, Pid} -> - case wolff_client:check_connectivity(Pid) of - ok -> ?status_connected; - {error, Error} -> {?status_connecting, State, Error} - end; - {error, _Reason} -> - ?status_connecting + %% Note: we must avoid returning `?status_disconnected' here if the connector ever was + %% connected. If the connector ever connected, wolff producers might have been + %% sucessfully started, and returning `?status_disconnected' will make resource + %% manager try to restart the producers / connector, thus potentially dropping data + %% held in wolff producer's replayq. + case check_client_connectivity(ClientId) of + ok -> + ?status_connected; + {error, {find_client, _Error}} -> + ?status_connecting; + {error, {connectivity, Error}} -> + {?status_connecting, State, Error} end. on_get_channel_status( @@ -496,13 +513,19 @@ on_get_channel_status( installed_bridge_v2s := Channels } = _State ) -> + %% Note: we must avoid returning `?status_disconnected' here. Returning + %% `?status_disconnected' will make resource manager try to restart the producers / + %% connector, thus potentially dropping data held in wolff producer's replayq. The + %% only exception is if the topic does not exist ("unhealthy target"). #{kafka_topic := KafkaTopic} = maps:get(ChannelId, Channels), try ok = check_topic_and_leader_connections(ClientId, KafkaTopic), ?status_connected catch - throw:#{reason := restarting} -> - ?status_connecting + throw:{unhealthy_target, Msg} -> + throw({unhealthy_target, Msg}); + K:E -> + {?status_connecting, {K, E}} end. check_topic_and_leader_connections(ClientId, KafkaTopic) -> @@ -524,6 +547,21 @@ check_topic_and_leader_connections(ClientId, KafkaTopic) -> }) end. +-spec check_client_connectivity(wolff:client_id()) -> + ok | {error, {connectivity | find_client, term()}}. +check_client_connectivity(ClientId) -> + case wolff_client_sup:find_client(ClientId) of + {ok, Pid} -> + case wolff_client:check_connectivity(Pid) of + ok -> + ok; + {error, Error} -> + {error, {connectivity, Error}} + end; + {error, Reason} -> + {error, {find_client, Reason}} + end. + check_if_healthy_leaders(ClientId, ClientPid, KafkaTopic) when is_pid(ClientPid) -> Leaders = case wolff_client:get_leader_connections(ClientPid, KafkaTopic) of diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 2ad0504b4..2913e178a 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -35,6 +36,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "toxiproxy.emqx.net"), + KafkaPort = list_to_integer(os:getenv("KAFKA_PLAIN_PORT", "9292")), + ProxyName = "kafka_plain", + DirectKafkaHost = os:getenv("KAFKA_DIRECT_PLAIN_HOST", "kafka-1.emqx.net"), + DirectKafkaPort = list_to_integer(os:getenv("KAFKA_DIRECT_PLAIN_PORT", "9092")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), Apps = emqx_cth_suite:start( [ emqx, @@ -50,17 +59,34 @@ init_per_suite(Config) -> ), {ok, _} = emqx_common_test_http:create_default_app(), emqx_bridge_kafka_impl_producer_SUITE:wait_until_kafka_is_up(), - [{apps, Apps} | Config]. + [ + {apps, Apps}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {proxy_name, ProxyName}, + {kafka_host, KafkaHost}, + {kafka_port, KafkaPort}, + {direct_kafka_host, DirectKafkaHost}, + {direct_kafka_port, DirectKafkaPort} + | Config + ]. end_per_suite(Config) -> Apps = ?config(apps, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), emqx_cth_suite:stop(Apps), ok. init_per_testcase(_TestCase, Config) -> Config. -end_per_testcase(_TestCase, _Config) -> +end_per_testcase(_TestCase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + emqx_bridge_v2_testlib:delete_all_bridges_and_connectors(), emqx_common_test_helpers:call_janitor(60_000), ok. @@ -69,6 +95,13 @@ end_per_testcase(_TestCase, _Config) -> %%------------------------------------------------------------------------------------- check_send_message_with_bridge(BridgeName) -> + #{offset := Offset, payload := Payload} = send_message(BridgeName), + %% ###################################### + %% Check if message is sent to Kafka + %% ###################################### + check_kafka_message_payload(Offset, Payload). + +send_message(ActionName) -> %% ###################################### %% Create Kafka message %% ###################################### @@ -84,11 +117,8 @@ check_send_message_with_bridge(BridgeName) -> %% ###################################### %% Send message %% ###################################### - emqx_bridge_v2:send_message(?TYPE, BridgeName, Msg, #{}), - %% ###################################### - %% Check if message is sent to Kafka - %% ###################################### - check_kafka_message_payload(Offset, Payload). + emqx_bridge_v2:send_message(?TYPE, ActionName, Msg, #{}), + #{offset => Offset, payload => Payload}. resolve_kafka_offset() -> KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(), @@ -106,6 +136,14 @@ check_kafka_message_payload(Offset, ExpectedPayload) -> {ok, {_, [KafkaMsg0]}} = brod:fetch(Hosts, KafkaTopic, Partition, Offset), ?assertMatch(#kafka_message{value = ExpectedPayload}, KafkaMsg0). +action_config(ConnectorName) -> + action_config(ConnectorName, _Overrides = #{}). + +action_config(ConnectorName, Overrides) -> + Cfg0 = bridge_v2_config(ConnectorName), + Cfg1 = emqx_utils_maps:rename(<<"kafka">>, <<"parameters">>, Cfg0), + emqx_utils_maps:deep_merge(Cfg1, Overrides). + bridge_v2_config(ConnectorName) -> #{ <<"connector">> => ConnectorName, @@ -131,7 +169,9 @@ bridge_v2_config(ConnectorName) -> <<"query_mode">> => <<"sync">>, <<"required_acks">> => <<"all_isr">>, <<"sync_query_timeout">> => <<"5s">>, - <<"topic">> => emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition() + <<"topic">> => list_to_binary( + emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition() + ) }, <<"local_topic">> => <<"kafka_t/#">>, <<"resource_opts">> => #{ @@ -140,32 +180,37 @@ bridge_v2_config(ConnectorName) -> }. connector_config() -> - #{ - <<"authentication">> => <<"none">>, - <<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()), - <<"connect_timeout">> => <<"5s">>, - <<"enable">> => true, - <<"metadata_request_timeout">> => <<"5s">>, - <<"min_metadata_refresh_interval">> => <<"3s">>, - <<"socket_opts">> => - #{ - <<"recbuf">> => <<"1024KB">>, - <<"sndbuf">> => <<"1024KB">>, - <<"tcp_keepalive">> => <<"none">> - }, - <<"ssl">> => - #{ - <<"ciphers">> => [], - <<"depth">> => 10, - <<"enable">> => false, - <<"hibernate_after">> => <<"5s">>, - <<"log_level">> => <<"notice">>, - <<"reuse_sessions">> => true, - <<"secure_renegotiate">> => true, - <<"verify">> => <<"verify_peer">>, - <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] - } - }. + connector_config(_Overrides = #{}). + +connector_config(Overrides) -> + Defaults = + #{ + <<"authentication">> => <<"none">>, + <<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()), + <<"connect_timeout">> => <<"5s">>, + <<"enable">> => true, + <<"metadata_request_timeout">> => <<"5s">>, + <<"min_metadata_refresh_interval">> => <<"3s">>, + <<"socket_opts">> => + #{ + <<"recbuf">> => <<"1024KB">>, + <<"sndbuf">> => <<"1024KB">>, + <<"tcp_keepalive">> => <<"none">> + }, + <<"ssl">> => + #{ + <<"ciphers">> => [], + <<"depth">> => 10, + <<"enable">> => false, + <<"hibernate_after">> => <<"5s">>, + <<"log_level">> => <<"notice">>, + <<"reuse_sessions">> => true, + <<"secure_renegotiate">> => true, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.3">>, <<"tlsv1.2">>] + } + }, + emqx_utils_maps:deep_merge(Defaults, Overrides). kafka_hosts_string() -> KafkaHost = os:getenv("KAFKA_PLAIN_HOST", "kafka-1.emqx.net"), @@ -350,13 +395,13 @@ t_bad_url(_Config) -> {ok, #{ resource_data := #{ - status := connecting, + status := ?status_disconnected, error := [#{reason := unresolvable_hostname}] } }}, emqx_connector:lookup(?TYPE, ConnectorName) ), - ?assertMatch({ok, #{status := connecting}}, emqx_bridge_v2:lookup(?TYPE, ActionName)), + ?assertMatch({ok, #{status := ?status_disconnected}}, emqx_bridge_v2:lookup(?TYPE, ActionName)), ok. t_parameters_key_api_spec(_Config) -> @@ -383,3 +428,153 @@ t_http_api_get(_Config) -> emqx_bridge_testlib:list_bridges_api() ), ok. + +t_create_connector_while_connection_is_down(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + KafkaHost = ?config(kafka_host, Config), + KafkaPort = ?config(kafka_port, Config), + Host = iolist_to_binary([KafkaHost, ":", integer_to_binary(KafkaPort)]), + ?check_trace( + begin + Type = ?TYPE, + ConnectorConfig = connector_config(#{ + <<"bootstrap_hosts">> => Host, + <<"resource_opts">> => + #{<<"health_check_interval">> => <<"500ms">>} + }), + ConnectorName = <<"c1">>, + ConnectorId = emqx_connector_resource:resource_id(Type, ConnectorName), + ConnectorParams = [ + {connector_config, ConnectorConfig}, + {connector_name, ConnectorName}, + {connector_type, Type} + ], + ActionName = ConnectorName, + ActionId = emqx_bridge_v2:id(?TYPE, ActionName, ConnectorName), + ActionConfig = action_config( + ConnectorName + ), + ActionParams = [ + {action_config, ActionConfig}, + {action_name, ActionName}, + {action_type, Type} + ], + Disconnected = atom_to_binary(?status_disconnected), + %% Initially, the connection cannot be stablished. Messages are not buffered, + %% hence the status is `?status_disconnected'. + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + {ok, {{_, 201, _}, _, #{<<"status">> := Disconnected}}} = + emqx_bridge_v2_testlib:create_connector_api(ConnectorParams), + {ok, {{_, 201, _}, _, #{<<"status">> := Disconnected}}} = + emqx_bridge_v2_testlib:create_action_api(ActionParams), + #{offset := Offset1} = send_message(ActionName), + #{offset := Offset2} = send_message(ActionName), + #{offset := Offset3} = send_message(ActionName), + ?assertEqual([Offset1], lists:usort([Offset1, Offset2, Offset3])), + ?assertEqual(3, emqx_resource_metrics:matched_get(ActionId)), + ?assertEqual(3, emqx_resource_metrics:failed_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:queuing_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:inflight_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:dropped_get(ActionId)), + ok + end), + %% Let the connector and action recover + Connected = atom_to_binary(?status_connected), + ?retry( + _Sleep0 = 1_100, + _Attempts0 = 10, + begin + _ = emqx_resource:health_check(ConnectorId), + _ = emqx_resource:health_check(ActionId), + ?assertMatch( + {ok, #{ + status := ?status_connected, + resource_data := + #{ + status := ?status_connected, + added_channels := + #{ + ActionId := #{ + status := ?status_connected + } + } + } + }}, + emqx_bridge_v2:lookup(Type, ActionName), + #{action_id => ActionId} + ), + ?assertMatch( + {ok, {{_, 200, _}, _, #{<<"status">> := Connected}}}, + emqx_bridge_v2_testlib:get_action_api(ActionParams) + ) + end + ), + %% Now the connection drops again; this time, status should be + %% `?status_connecting' to avoid destroying wolff_producers and their replayq + %% buffers. + Connecting = atom_to_binary(?status_connecting), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?retry( + _Sleep0 = 1_100, + _Attempts0 = 10, + begin + _ = emqx_resource:health_check(ConnectorId), + _ = emqx_resource:health_check(ActionId), + ?assertMatch( + {ok, #{ + status := ?status_connecting, + resource_data := + #{ + status := ?status_connecting, + added_channels := + #{ + ActionId := #{ + status := ?status_connecting + } + } + } + }}, + emqx_bridge_v2:lookup(Type, ActionName), + #{action_id => ActionId} + ), + ?assertMatch( + {ok, {{_, 200, _}, _, #{<<"status">> := Connecting}}}, + emqx_bridge_v2_testlib:get_action_api(ActionParams) + ) + end + ), + %% This should get enqueued by wolff producers. + spawn_link(fun() -> send_message(ActionName) end), + PreviousMatched = 3, + PreviousFailed = 3, + ?retry( + _Sleep2 = 100, + _Attempts2 = 10, + ?assertEqual(PreviousMatched + 1, emqx_resource_metrics:matched_get(ActionId)) + ), + ?assertEqual(PreviousFailed, emqx_resource_metrics:failed_get(ActionId)), + ?assertEqual(1, emqx_resource_metrics:queuing_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:inflight_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:dropped_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:success_get(ActionId)), + ok + end), + ?retry( + _Sleep2 = 600, + _Attempts2 = 20, + begin + _ = emqx_resource:health_check(ConnectorId), + _ = emqx_resource:health_check(ActionId), + ?assertEqual(1, emqx_resource_metrics:success_get(ActionId), #{ + metrics => emqx_bridge_v2:get_metrics(Type, ActionName) + }), + ok + end + ), + ok + end, + [] + ), + ok. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index cbef0dda8..716626bdf 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 252b8ff00..b1d110d36 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index 614747254..fafd49f05 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl index 5492bb2a8..5b9c33fbb 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl @@ -73,7 +73,7 @@ check_atom_key(Conf) when is_map(Conf) -> %% erlfmt-ignore pulsar_producer_hocon() -> -""" +" bridges.pulsar_producer.my_producer { enable = true servers = \"localhost:6650\" @@ -90,4 +90,4 @@ bridges.pulsar_producer.my_producer { server_name_indication = \"auto\" } } -""". +". diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index 7e32b5a89..2e1ec3444 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 49942065a..6887582b3 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -12,7 +12,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --export([roots/0, fields/1, desc/1, connector_examples/1]). +-export([namespace/0, roots/0, fields/1, desc/1, connector_examples/1]). %% `emqx_resource' API -export([ @@ -44,6 +44,8 @@ %% ------------------------------------------------------------------------------------------------- %% api +namespace() -> "syskeeper_forwarder". + connector_examples(Method) -> [ #{ diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 5bc330afa..756a5ec30 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -66,6 +66,7 @@ -boot_mnesia({mnesia, [boot]}). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_conf.hrl"). -ifdef(TEST). @@ -384,6 +385,7 @@ catch_up(State) -> catch_up(State, false). catch_up(#{node := Node, retry_interval := RetryMs, is_leaving := false} = State, SkipResult) -> case transaction(fun ?MODULE:read_next_mfa/1, [Node]) of {atomic, caught_up} -> + ?tp(cluster_rpc_caught_up, #{}), ?TIMEOUT; {atomic, {still_lagging, NextId, MFA}} -> {Succeed, _} = apply_mfa(NextId, MFA, ?APPLY_KIND_REPLICATE), diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 3856a882c..7f495a3cd 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.31"}, + {vsn, "0.1.32"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 0925141de..0d2ee72e4 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -306,7 +306,7 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> ShortName = short_name(FullName), case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of true -> - io:format(standard_error, "WARN: no_namespace_for: ~s~n", [FullName]); + error({no_namespace, FullName, S}); false -> ok end, diff --git a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl index 096136651..2cb699036 100644 --- a/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_logger_SUITE.erl @@ -24,7 +24,7 @@ %% erlfmt-ignore -define(BASE_CONF, - """ + " log { console { enable = true @@ -36,7 +36,7 @@ path = \"log/emqx.log\" } } - """). + "). all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 4fca88a00..22f8c5575 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -20,7 +20,7 @@ %% erlfmt-ignore -define(BASE_CONF, - """ + " node { name = \"emqx1@127.0.0.1\" cookie = \"emqxsecretcookie\" @@ -34,7 +34,7 @@ static.seeds = ~p core_nodes = ~p } - """). + "). array_nodes_test() -> ensure_acl_conf(), @@ -70,7 +70,7 @@ array_nodes_test() -> %% erlfmt-ignore -define(OUTDATED_LOG_CONF, - """ + " log.console_handler { burst_limit { enable = true @@ -124,7 +124,7 @@ log.file_handlers { time_offset = \"+01:00\" } } - """ + " ). -define(FORMATTER(TimeOffset), {emqx_logger_textfmt, #{ @@ -196,7 +196,7 @@ validate_log(Conf) -> %% erlfmt-ignore -define(FILE_LOG_BASE_CONF, - """ + " log.file.default { enable = true file = \"log/xx-emqx.log\" @@ -206,7 +206,7 @@ validate_log(Conf) -> rotation_size = ~s time_offset = \"+01:00\" } - """ + " ). file_log_infinity_rotation_size_test_() -> @@ -249,7 +249,7 @@ file_log_infinity_rotation_size_test_() -> %% erlfmt-ignore -define(KERNEL_LOG_CONF, - """ + " log.console { enable = true formatter = text @@ -269,7 +269,7 @@ file_log_infinity_rotation_size_test_() -> enable = true file = \"log/my-emqx.log\" } - """ + " ). log_test() -> @@ -279,7 +279,7 @@ log_test() -> log_rotation_count_limit_test() -> ensure_acl_conf(), Format = - """ + " log.file { enable = true path = \"log/emqx.log\" @@ -288,7 +288,7 @@ log_rotation_count_limit_test() -> rotation = {count = ~w} rotation_size = \"1024MB\" } - """, + ", BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), lists:foreach(fun({Conf, Count}) -> Conf0 = <>, @@ -320,7 +320,7 @@ log_rotation_count_limit_test() -> %% erlfmt-ignore -define(BASE_AUTHN_ARRAY, - """ + " authentication = [ {backend = \"http\" body {password = \"${password}\", username = \"${username}\"} @@ -335,7 +335,7 @@ log_rotation_count_limit_test() -> url = \"~ts\" } ] - """ + " ). -define(ERROR(Error), @@ -396,13 +396,13 @@ authn_validations_test() -> %% erlfmt-ignore -define(LISTENERS, - """ + " listeners.ssl.default.bind = 9999 listeners.wss.default.bind = 9998 listeners.wss.default.ssl_options.cacertfile = \"mytest/certs/cacert.pem\" listeners.wss.new.bind = 9997 listeners.wss.new.websocket.mqtt_path = \"/my-mqtt\" - """ + " ). listeners_test() -> diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 99f76f635..7150d1e7a 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.34"}, + {vsn, "0.1.35"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index b31fb4f74..f25fe9b7e 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -137,7 +137,7 @@ param_path_id() -> #{ in => path, required => true, - example => <<"webhook:webhook_example">>, + example => <<"http:my_http_connector">>, desc => ?DESC("desc_param_path_id") } )}. @@ -158,17 +158,7 @@ connector_info_array_example(Method) -> lists:map(fun(#{value := Config}) -> Config end, maps:values(connector_info_examples(Method))). connector_info_examples(Method) -> - maps:merge( - #{}, - emqx_enterprise_connector_examples(Method) - ). - --if(?EMQX_RELEASE_EDITION == ee). -emqx_enterprise_connector_examples(Method) -> - emqx_connector_ee_schema:examples(Method). --else. -emqx_enterprise_connector_examples(_Method) -> #{}. --endif. + emqx_connector_schema:examples(Method). schema("/connectors") -> #{ diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index bc648a102..ff2790481 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -49,6 +49,8 @@ get_channels/2 ]). +-export([parse_url/1]). + -callback connector_config(ParsedConfig) -> ParsedConfig when @@ -77,8 +79,10 @@ connector_impl_module(_ConnectorType) -> -endif. -connector_to_resource_type_ce(_ConnectorType) -> - no_bridge_v2_for_c2_so_far. +connector_to_resource_type_ce(http) -> + emqx_bridge_http_connector; +connector_to_resource_type_ce(ConnectorType) -> + error({no_bridge_v2, ConnectorType}). resource_id(ConnectorId) when is_binary(ConnectorId) -> <<"connector:", ConnectorId/binary>>. @@ -271,13 +275,11 @@ remove(Type, Name, _Conf, _Opts) -> %% convert connector configs to what the connector modules want parse_confs( - <<"webhook">>, + <<"http">>, _Name, #{ url := Url, - method := Method, - headers := Headers, - max_retries := Retry + headers := Headers } = Conf ) -> Url1 = bin(Url), @@ -290,20 +292,14 @@ parse_confs( Reason1 = emqx_utils:readable_error_msg(Reason), invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) end, - RequestTTL = emqx_utils_maps:deep_get( - [resource_opts, request_ttl], - Conf - ), Conf#{ base_url => BaseUrl1, request => #{ path => Path, - method => Method, - body => maps:get(body, Conf, undefined), headers => Headers, - request_ttl => RequestTTL, - max_retries => Retry + body => undefined, + method => undefined } }; parse_confs(<<"iotdb">>, Name, Conf) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index 1d16fd1a1..1ca6e4a5d 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -15,7 +15,8 @@ -export([ api_schemas/1, fields/1, - examples/1 + %%examples/1 + schema_modules/0 ]). resource_type(Type) when is_binary(Type) -> @@ -151,18 +152,6 @@ connector_structs() -> )} ]. -examples(Method) -> - MergeFun = - fun(Example, Examples) -> - maps:merge(Examples, Example) - end, - Fun = - fun(Module, Examples) -> - ConnectorExamples = erlang:apply(Module, connector_examples, [Method]), - lists:foldl(MergeFun, Examples, ConnectorExamples) - end, - lists:foldl(Fun, #{}, schema_modules()). - schema_modules() -> [ emqx_bridge_azure_event_hub, diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 9759c8f0a..d4f82d474 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -42,6 +42,8 @@ -export([resource_opts_fields/0, resource_opts_fields/1]). +-export([examples/1]). + -if(?EMQX_RELEASE_EDITION == ee). enterprise_api_schemas(Method) -> %% We *must* do this to ensure the module is really loaded, especially when we use @@ -71,6 +73,40 @@ enterprise_fields_connectors() -> []. -endif. +api_schemas(Method) -> + [ + %% We need to map the `type' field of a request (binary) to a + %% connector schema module. + api_ref(emqx_bridge_http_schema, <<"http">>, Method ++ "_connector") + ]. + +api_ref(Module, Type, Method) -> + {Type, ref(Module, Method)}. + +examples(Method) -> + MergeFun = + fun(Example, Examples) -> + maps:merge(Examples, Example) + end, + Fun = + fun(Module, Examples) -> + ConnectorExamples = erlang:apply(Module, connector_examples, [Method]), + lists:foldl(MergeFun, Examples, ConnectorExamples) + end, + lists:foldl(Fun, #{}, schema_modules()). + +-if(?EMQX_RELEASE_EDITION == ee). +schema_modules() -> + [emqx_bridge_http_schema] ++ emqx_connector_ee_schema:schema_modules(). +-else. +schema_modules() -> + [emqx_bridge_http_schema]. +-endif. + +%% @doc Return old bridge(v1) and/or connector(v2) type +%% from the latest connector type name. +connector_type_to_bridge_types(http) -> + [webhook, http]; connector_type_to_bridge_types(azure_event_hub_producer) -> [azure_event_hub_producer]; connector_type_to_bridge_types(confluent_producer) -> @@ -323,8 +359,9 @@ post_request() -> api_schema("post"). api_schema(Method) -> + CE = api_schemas(Method), EE = enterprise_api_schemas(Method), - hoconsc:union(connector_api_union(EE)). + hoconsc:union(connector_api_union(CE ++ EE)). connector_api_union(Refs) -> Index = maps:from_list(Refs), @@ -369,7 +406,17 @@ roots() -> end. fields(connectors) -> - [] ++ enterprise_fields_connectors(); + [ + {http, + mk( + hoconsc:map(name, ref(emqx_bridge_http_schema, "config_connector")), + #{ + alias => [webhook], + desc => <<"HTTP Connector Config">>, + required => false + } + )} + ] ++ enterprise_fields_connectors(); fields("node_status") -> [ node_name(), diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 97691c6cd..9474d868f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.30"}, + {vsn, "5.0.31"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [ diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src index ec8e6cd3f..acc5e6cbd 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_rbac, [ {description, "EMQX Dashboard RBAC"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index 71788947b..19f3bf552 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, diff --git a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl index 077df28d0..5c451206d 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_replication_layer_meta.erl @@ -192,9 +192,9 @@ sites() -> {ok, node()} | {error, no_leader_for_shard}. shard_leader(DB, Shard) -> case mnesia:dirty_read(?SHARD_TAB, {DB, Shard}) of - [#?SHARD_TAB{leader = Leader}] -> + [#?SHARD_TAB{leader = Leader}] when Leader =/= undefined -> {ok, Leader}; - [] -> + _ -> {error, no_leader_for_shard} end. @@ -314,7 +314,7 @@ ensure_tables() -> {rlog_shard, ?SHARD}, {majority, Majority}, {type, ordered_set}, - {storage, ram_copies}, + {storage, disc_copies}, {record_name, ?SHARD_TAB}, {attributes, record_info(fields, ?SHARD_TAB)} ]), diff --git a/apps/emqx_durable_storage/src/emqx_durable_storage.app.src b/apps/emqx_durable_storage/src/emqx_durable_storage.app.src index 2bce4ff8e..8d868bc75 100644 --- a/apps/emqx_durable_storage/src/emqx_durable_storage.app.src +++ b/apps/emqx_durable_storage/src/emqx_durable_storage.app.src @@ -2,7 +2,7 @@ {application, emqx_durable_storage, [ {description, "Message persistence and subscription replays for EMQX"}, % strict semver, bump manually! - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, rocksdb, gproc, mria, emqx_utils]}, diff --git a/apps/emqx_enterprise/src/emqx_enterprise.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src index 06bc500f4..d7bcb1fd5 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ {application, emqx_enterprise, [ {description, "EMQX Enterprise Edition"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index df681b00f..81a2e65ed 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.27"}, + {vsn, "0.1.28"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]}, diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl index 743c74e70..4580cc087 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl @@ -4,12 +4,18 @@ -module(emqx_gbt32960_schema). +-behaviour(hocon_schema). + -include("emqx_gbt32960.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1]). + +namespace() -> "gateway_gbt32960". + +roots() -> []. fields(gbt32960) -> [ diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index 97a6e04a1..f4ab5bd24 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, {env, []}, diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_schema.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_schema.erl index 69fc3aa78..d4609962c 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_schema.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_schema.erl @@ -10,7 +10,11 @@ -define(DEFAULT_MOUNTPOINT, <<"ocpp/">>). %% config schema provides --export([fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1]). + +namespace() -> "gateway_ocpp". + +roots() -> []. fields(ocpp) -> [ diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src index 6913b2c5f..08214aee2 100644 --- a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_stomp, [ {description, "Stomp Gateway"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index 774f11bd4..546c9975c 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_license/src/emqx_license.app.src b/apps/emqx_license/src/emqx_license.app.src index eb639d164..8d11c6522 100644 --- a/apps/emqx_license/src/emqx_license.app.src +++ b/apps/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_license/src/emqx_license_checker.erl b/apps/emqx_license/src/emqx_license_checker.erl index da777ff84..198814fb9 100644 --- a/apps/emqx_license/src/emqx_license_checker.erl +++ b/apps/emqx_license/src/emqx_license_checker.erl @@ -5,12 +5,15 @@ -module(emqx_license_checker). -include("emqx_license.hrl"). +-include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -behaviour(gen_server). --define(CHECK_INTERVAL, 5000). --define(EXPIRY_ALARM_CHECK_INTERVAL, 24 * 60 * 60). +-define(CHECK_INTERVAL, timer:seconds(5)). +-define(REFRESH_INTERVAL, timer:minutes(2)). +-define(EXPIRY_ALARM_CHECK_INTERVAL, timer:hours(24)). + -define(OK(EXPR), try _ = begin @@ -56,7 +59,7 @@ start_link(LicenseFetcher) -> start_link(LicenseFetcher, CheckInterval) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [LicenseFetcher, CheckInterval], []). --spec update(emqx_license_parser:license()) -> ok. +-spec update(emqx_license_parser:license()) -> map(). update(License) -> gen_server:call(?MODULE, {update, License}, infinity). @@ -94,15 +97,18 @@ init([LicenseFetcher, CheckInterval]) -> check_license_interval => CheckInterval, license => License }), - State = ensure_check_expiry_timer(State0), + State1 = ensure_refresh_timer(State0), + State = ensure_check_expiry_timer(State1), {ok, State}; {error, Reason} -> {stop, Reason} end. -handle_call({update, License}, _From, State) -> +handle_call({update, License}, _From, #{license := Old} = State) -> ok = expiry_early_alarm(License), - {reply, check_license(License), State#{license => License}}; + State1 = ensure_refresh_timer(State), + ok = log_new_license(Old, License), + {reply, check_license(License), State1#{license => License}}; handle_call(dump, _From, #{license := License} = State) -> {reply, emqx_license_parser:dump(License), State}; handle_call(purge, _From, State) -> @@ -123,6 +129,10 @@ handle_info(check_expiry_alarm, #{license := License} = State) -> ok = expiry_early_alarm(License), NewState = ensure_check_expiry_timer(State), {noreply, NewState}; +handle_info(refresh, State0) -> + State1 = refresh(State0), + NewState = ensure_refresh_timer(State1), + {noreply, NewState}; handle_info(_Msg, State) -> {noreply, State}. @@ -130,22 +140,59 @@ handle_info(_Msg, State) -> %% Private functions %%------------------------------------------------------------------------------ +refresh(#{license := #{source := <<"file://", _/binary>> = Source} = License} = State) -> + case emqx_license_parser:parse(Source) of + {ok, License} -> + ?tp(emqx_license_refresh_no_change, #{}), + %% no change + State; + {ok, NewLicense} -> + ok = log_new_license(License, NewLicense), + %% ensure alarm is set or cleared + ok = expiry_early_alarm(NewLicense), + ?tp(emqx_license_refresh_changed, #{new_license => NewLicense}), + State#{license => NewLicense}; + {error, Reason} -> + ?tp( + error, + emqx_license_refresh_failed, + Reason#{continue_with_license => emqx_license_parser:summary(License)} + ), + State + end; +refresh(State) -> + State. + +log_new_license(Old, New) -> + ?SLOG(info, #{ + msg => "new_license_loaded", + old_license => emqx_license_parser:summary(Old), + new_license => emqx_license_parser:summary(New) + }). + ensure_check_license_timer(#{check_license_interval := CheckInterval} = State) -> - cancel_timer(State, timer), - State#{timer => erlang:send_after(CheckInterval, self(), check_license)}. + ok = cancel_timer(State, check_timer), + State#{check_timer => erlang:send_after(CheckInterval, self(), check_license)}. ensure_check_expiry_timer(State) -> - cancel_timer(State, expiry_alarm_timer), + ok = cancel_timer(State, expiry_alarm_timer), Ref = erlang:send_after(?EXPIRY_ALARM_CHECK_INTERVAL, self(), check_expiry_alarm), State#{expiry_alarm_timer => Ref}. +%% refresh is to work with file:// license keys. +ensure_refresh_timer(State) -> + ok = cancel_timer(State, refresh_timer), + Ref = erlang:send_after(?REFRESH_INTERVAL, self(), refresh), + State#{refresh_timer => Ref}. + cancel_timer(State, Key) -> - _ = - case maps:find(Key, State) of - {ok, Ref} when is_reference(Ref) -> erlang:cancel_timer(Ref); - _ -> ok - end, - ok. + case maps:find(Key, State) of + {ok, Ref} when is_reference(Ref) -> + _ = erlang:cancel_timer(Ref), + ok; + _ -> + ok + end. check_license(License) -> DaysLeft = days_left(License), diff --git a/apps/emqx_license/src/emqx_license_http_api.erl b/apps/emqx_license/src/emqx_license_http_api.erl index 8f563300b..439632c75 100644 --- a/apps/emqx_license/src/emqx_license_http_api.erl +++ b/apps/emqx_license/src/emqx_license_http_api.erl @@ -54,7 +54,6 @@ schema("/license") -> ) } }, - %% TODO(5.x): It's a update action, should use PUT instead post => #{ tags => ?LICENSE_TAGS, summary => <<"Update license key">>, diff --git a/apps/emqx_license/src/emqx_license_parser.erl b/apps/emqx_license/src/emqx_license_parser.erl index 05625902a..88304a6db 100644 --- a/apps/emqx_license/src/emqx_license_parser.erl +++ b/apps/emqx_license/src/emqx_license_parser.erl @@ -32,7 +32,14 @@ -type license_type() :: ?OFFICIAL | ?TRIAL. --type license() :: #{module := module(), data := license_data()}. +-type license() :: #{ + %% the parser module which parsed the license + module := module(), + %% the parse result + data := license_data(), + %% the source of the license, e.g. "file://path/to/license/file" or "******" for license key + source := binary() +}. -export_type([ license_data/0, @@ -45,12 +52,19 @@ parse/1, parse/2, dump/1, + summary/1, customer_type/1, license_type/1, expiry_date/1, max_connections/1 ]). +%% for testing purpose +-export([ + default/0, + pubkey/0 +]). + %%-------------------------------------------------------------------- %% Behaviour %%-------------------------------------------------------------------- @@ -59,6 +73,9 @@ -callback dump(license_data()) -> list({atom(), term()}). +%% provide a summary map for logging purposes +-callback summary(license_data()) -> map(). + -callback customer_type(license_data()) -> customer_type(). -callback license_type(license_data()) -> license_type(). @@ -71,19 +88,37 @@ %% API %%-------------------------------------------------------------------- --ifdef(TEST). --spec parse(string() | binary()) -> {ok, license()} | {error, term()}. -parse(Content) -> - PubKey = persistent_term:get(emqx_license_test_pubkey, ?PUBKEY), - parse(Content, PubKey). --else. --spec parse(string() | binary()) -> {ok, license()} | {error, term()}. -parse(Content) -> - parse(Content, ?PUBKEY). --endif. +pubkey() -> ?PUBKEY. +default() -> emqx_license_schema:default_license(). -parse(Content, Pem) -> - [PemEntry] = public_key:pem_decode(Pem), +%% @doc Parse license key. +%% If the license key is prefixed with "file://path/to/license/file", +%% then the license key is read from the file. +-spec parse(default | string() | binary()) -> {ok, license()} | {error, map()}. +parse(Content) -> + parse(to_bin(Content), ?MODULE:pubkey()). + +parse(<<"default">>, PubKey) -> + parse(?MODULE:default(), PubKey); +parse(<<"file://", Path/binary>> = FileKey, PubKey) -> + case file:read_file(Path) of + {ok, Content} -> + case parse(Content, PubKey) of + {ok, License} -> + {ok, License#{source => FileKey}}; + {error, Reason} -> + {error, Reason#{ + license_file => Path + }} + end; + {error, Reason} -> + {error, #{ + license_file => Path, + read_error => Reason + }} + end; +parse(Content, PubKey) -> + [PemEntry] = public_key:pem_decode(PubKey), Key = public_key:pem_entry_decode(PemEntry), do_parse(iolist_to_binary(Content), Key, ?LICENSE_PARSE_MODULES, []). @@ -91,6 +126,10 @@ parse(Content, Pem) -> dump(#{module := Module, data := LicenseData}) -> Module:dump(LicenseData). +-spec summary(license()) -> map(). +summary(#{module := Module, data := Data}) -> + Module:summary(Data). + -spec customer_type(license()) -> customer_type(). customer_type(#{module := Module, data := LicenseData}) -> Module:customer_type(LicenseData). @@ -112,14 +151,21 @@ max_connections(#{module := Module, data := LicenseData}) -> %%-------------------------------------------------------------------- do_parse(_Content, _Key, [], Errors) -> - {error, lists:reverse(Errors)}; + {error, #{parse_results => lists:reverse(Errors)}}; do_parse(Content, Key, [Module | Modules], Errors) -> try Module:parse(Content, Key) of {ok, LicenseData} -> - {ok, #{module => Module, data => LicenseData}}; + {ok, #{module => Module, data => LicenseData, source => <<"******">>}}; {error, Error} -> - do_parse(Content, Key, Modules, [{Module, Error} | Errors]) + do_parse(Content, Key, Modules, [#{module => Module, error => Error} | Errors]) catch _Class:Error:Stacktrace -> - do_parse(Content, Key, Modules, [{Module, {Error, Stacktrace}} | Errors]) + do_parse(Content, Key, Modules, [ + #{module => Module, error => Error, stacktrace => Stacktrace} | Errors + ]) end. + +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(L) -> + iolist_to_binary(L). diff --git a/apps/emqx_license/src/emqx_license_parser_v20220101.erl b/apps/emqx_license/src/emqx_license_parser_v20220101.erl index 4b2d6dccd..decdc4822 100644 --- a/apps/emqx_license/src/emqx_license_parser_v20220101.erl +++ b/apps/emqx_license/src/emqx_license_parser_v20220101.erl @@ -21,6 +21,7 @@ -export([ parse/2, dump/1, + summary/1, customer_type/1, license_type/1, expiry_date/1, @@ -69,6 +70,21 @@ dump( {expiry, Expiry} ]. +summary( + #{ + deployment := Deployment, + date_start := DateStart, + max_connections := MaxConns + } = License +) -> + DateExpiry = expiry_date(License), + #{ + deployment => Deployment, + max_connections => MaxConns, + start_at => format_date(DateStart), + expiry_at => format_date(DateExpiry) + }. + customer_type(#{customer_type := CType}) -> CType. license_type(#{type := Type}) -> Type. @@ -85,17 +101,27 @@ max_connections(#{max_connections := MaxConns}) -> %% Private functions %%------------------------------------------------------------------------------ -do_parse(Content) -> +do_parse(Content0) -> try - [EncodedPayload, EncodedSignature] = binary:split(Content, <<".">>), - Payload = base64:decode(EncodedPayload), - Signature = base64:decode(EncodedSignature), - {ok, {Payload, Signature}} + Content = normalize(Content0), + do_parse2(Content) catch _:_ -> {error, bad_license_format} end. +do_parse2(<<>>) -> + {error, empty_string}; +do_parse2(Content) -> + [EncodedPayload, EncodedSignature] = binary:split(Content, <<".">>), + Payload = base64:decode(EncodedPayload), + Signature = base64:decode(EncodedSignature), + {ok, {Payload, Signature}}. + +%% drop whitespaces and newlines (CRLF) +normalize(Bin) -> + <<<> || <> <= Bin, C =/= $\s andalso C =/= $\n andalso C =/= $\r>>. + verify_signature(Payload, Signature, Key) -> public_key:verify(Payload, ?DIGEST_TYPE, Signature, Key). @@ -182,7 +208,7 @@ collect_fields(Fields) -> {FieldValues, []} -> {ok, maps:from_list(FieldValues)}; {_, Errors} -> - {error, lists:reverse(Errors)} + {error, maps:from_list(Errors)} end. format_date({Year, Month, Day}) -> diff --git a/apps/emqx_license/src/emqx_license_schema.erl b/apps/emqx_license/src/emqx_license_schema.erl index f2b91811e..aeced0ab0 100644 --- a/apps/emqx_license/src/emqx_license_schema.erl +++ b/apps/emqx_license/src/emqx_license_schema.erl @@ -38,8 +38,8 @@ tags() -> fields(key_license) -> [ {key, #{ - type => binary(), - default => default_license(), + type => hoconsc:union([default, binary()]), + default => <<"default">>, %% so it's not logged sensitive => true, required => true, @@ -72,10 +72,16 @@ check_license_watermark(Conf) -> undefined -> true; Low -> - High = hocon_maps:get("license.connection_high_watermark", Conf), - case High =/= undefined andalso High > Low of - true -> true; - false -> {bad_license_watermark, #{high => High, low => Low}} + case hocon_maps:get("license.connection_high_watermark", Conf) of + undefined -> + {bad_license_watermark, #{high => undefined, low => Low}}; + High -> + {ok, HighFloat} = emqx_schema:to_percent(High), + {ok, LowFloat} = emqx_schema:to_percent(Low), + case HighFloat > LowFloat of + true -> true; + false -> {bad_license_watermark, #{high => High, low => Low}} + end end end. diff --git a/apps/emqx_license/test/emqx_license_SUITE.erl b/apps/emqx_license/test/emqx_license_SUITE.erl index 69adabe76..4818ad9e6 100644 --- a/apps/emqx_license/test/emqx_license_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_SUITE.erl @@ -16,12 +16,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + emqx_license_test_lib:mock_parser(), _ = application:load(emqx_conf), emqx_config:save_schema_mod_and_names(emqx_license_schema), emqx_common_test_helpers:start_apps([emqx_license], fun set_special_configs/1), Config. end_per_suite(_) -> + emqx_license_test_lib:unmock_parser(), emqx_common_test_helpers:stop_apps([emqx_license]), ok. @@ -103,17 +105,7 @@ setup_test(TestCase, Config) when ), ok; (emqx_license) -> - LicensePath = filename:join(emqx_license:license_dir(), "emqx.lic"), - filelib:ensure_dir(LicensePath), - ok = file:write_file(LicensePath, LicenseKey), - LicConfig = #{type => file, file => LicensePath}, - emqx_config:put([license], LicConfig), - RawConfig = #{<<"type">> => file, <<"file">> => LicensePath}, - emqx_config:put_raw([<<"license">>], RawConfig), - ok = persistent_term:put( - emqx_license_test_pubkey, - emqx_license_test_lib:public_key_pem() - ), + set_special_configs(emqx_license), ok; (_) -> ok @@ -129,9 +121,9 @@ teardown_test(_TestCase, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{key => emqx_license_test_lib:default_license()}, + Config = #{key => default}, emqx_config:put([license], Config), - RawConfig = #{<<"key">> => emqx_license_test_lib:default_license()}, + RawConfig = #{<<"key">> => <<"default">>}, emqx_config:put_raw([<<"license">>], RawConfig); set_special_configs(_) -> ok. @@ -146,11 +138,11 @@ assert_on_nodes(Nodes, RunFun, CheckFun) -> t_update_value(_Config) -> ?assertMatch( - {error, [_ | _]}, + {error, #{parse_results := [_ | _]}}, emqx_license:update_key("invalid.license") ), - LicenseValue = emqx_license_test_lib:default_license(), + LicenseValue = emqx_license_test_lib:default_test_license(), ?assertMatch( {ok, #{}}, diff --git a/apps/emqx_license/test/emqx_license_checker_SUITE.erl b/apps/emqx_license/test/emqx_license_checker_SUITE.erl index a4ef1af6e..5733a09ce 100644 --- a/apps/emqx_license/test/emqx_license_checker_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_checker_SUITE.erl @@ -14,12 +14,14 @@ all() -> emqx_common_test_helpers:all(?MODULE). -init_per_suite(Config) -> +init_per_suite(CtConfig) -> _ = application:load(emqx_conf), + emqx_license_test_lib:mock_parser(), ok = emqx_common_test_helpers:start_apps([emqx_license], fun set_special_configs/1), - Config. + CtConfig. end_per_suite(_) -> + emqx_license_test_lib:unmock_parser(), ok = emqx_common_test_helpers:stop_apps([emqx_license]). init_per_testcase(t_default_limits, Config) -> @@ -35,7 +37,7 @@ end_per_testcase(_Case, _Config) -> ok. set_special_configs(emqx_license) -> - Config = #{key => emqx_license_test_lib:default_license()}, + Config = #{key => emqx_license_test_lib:default_test_license()}, emqx_config:put([license], Config); set_special_configs(_) -> ok. @@ -100,7 +102,7 @@ t_update(_Config) -> emqx_license_checker:limits() ). -t_update_by_timer(_Config) -> +t_check_by_timer(_Config) -> ?check_trace( begin ?wait_async_action( @@ -228,10 +230,111 @@ t_unknown_calls(_Config) -> some_msg = erlang:send(emqx_license_checker, some_msg), ?assertEqual(unknown, gen_server:call(emqx_license_checker, some_request)). +t_refresh_no_change(Config) when is_list(Config) -> + {ok, License} = write_test_license(Config, ?FUNCTION_NAME, 1, 111), + #{} = emqx_license_checker:update(License), + ?check_trace( + begin + ?wait_async_action( + begin + erlang:send( + emqx_license_checker, + refresh + ) + end, + #{?snk_kind := emqx_license_refresh_no_change}, + 1000 + ) + end, + fun(Trace) -> + ?assertMatch([_ | _], ?of_kind(emqx_license_refresh_no_change, Trace)) + end + ). + +t_refresh_change(Config) when is_list(Config) -> + {ok, License} = write_test_license(Config, ?FUNCTION_NAME, 1, 111), + #{} = emqx_license_checker:update(License), + {ok, License2} = write_test_license(Config, ?FUNCTION_NAME, 2, 222), + ?check_trace( + begin + ?wait_async_action( + begin + erlang:send( + emqx_license_checker, + refresh + ) + end, + #{?snk_kind := emqx_license_refresh_changed}, + 1000 + ) + end, + fun(Trace) -> + ?assertMatch( + [#{new_license := License2} | _], ?of_kind(emqx_license_refresh_changed, Trace) + ) + end + ). + +t_refresh_failure(Config) when is_list(Config) -> + Filename = test_license_file_name(Config, ?FUNCTION_NAME), + {ok, License} = write_test_license(Config, ?FUNCTION_NAME, 1, 111), + Summary = emqx_license_parser:summary(License), + #{} = emqx_license_checker:update(License), + ok = file:write_file(Filename, <<"invalid license">>), + ?check_trace( + begin + ?wait_async_action( + begin + erlang:send( + emqx_license_checker, + refresh + ) + end, + #{?snk_kind := emqx_license_refresh_failed}, + 1000 + ) + end, + fun(Trace) -> + ?assertMatch( + [#{continue_with_license := Summary} | _], + ?of_kind(emqx_license_refresh_failed, Trace) + ) + end + ). + %%------------------------------------------------------------------------------ %% Tests %%------------------------------------------------------------------------------ +write_test_license(Config, Name, ExpireInDays, Connections) -> + {NowDate, _} = calendar:universal_time(), + DateTomorrow = calendar:gregorian_days_to_date( + calendar:date_to_gregorian_days(NowDate) + ExpireInDays + ), + Fields = [ + "220111", + "1", + "0", + "Foo", + "contact@foo.com", + "bar", + format_date(DateTomorrow), + "1", + integer_to_list(Connections) + ], + FileName = test_license_file_name(Config, Name), + ok = write_license_file(FileName, Fields), + emqx_license_parser:parse(<<"file://", FileName/binary>>). + +test_license_file_name(Config, Name) -> + Dir = ?config(data_dir, Config), + iolist_to_binary(filename:join(Dir, atom_to_list(Name) ++ ".lic")). + +write_license_file(FileName, Fields) -> + EncodedLicense = emqx_license_test_lib:make_license(Fields), + ok = filelib:ensure_dir(FileName), + ok = file:write_file(FileName, EncodedLicense). + mk_license(Fields) -> EncodedLicense = emqx_license_test_lib:make_license(Fields), {ok, License} = emqx_license_parser:parse( diff --git a/apps/emqx_license/test/emqx_license_cli_SUITE.erl b/apps/emqx_license/test/emqx_license_cli_SUITE.erl index 1c6282262..ed6593aac 100644 --- a/apps/emqx_license/test/emqx_license_cli_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_cli_SUITE.erl @@ -24,15 +24,12 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> - ok = persistent_term:put( - emqx_license_test_pubkey, - emqx_license_test_lib:public_key_pem() - ), + emqx_license_test_lib:mock_parser(), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), Config. end_per_testcase(_Case, _Config) -> - persistent_term:erase(emqx_license_test_pubkey), + emqx_license_test_lib:unmock_parser(), ok. set_special_configs(emqx_license) -> diff --git a/apps/emqx_license/test/emqx_license_http_api_SUITE.erl b/apps/emqx_license/test/emqx_license_http_api_SUITE.erl index 4ee0c8c8e..ad16d75c9 100644 --- a/apps/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -19,6 +19,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + emqx_license_test_lib:mock_parser(), _ = application:load(emqx_conf), emqx_config:save_schema_mod_and_names(emqx_license_schema), emqx_common_test_helpers:start_apps([emqx_license, emqx_dashboard], fun set_special_configs/1), @@ -31,7 +32,7 @@ end_per_suite(_) -> emqx_config:put([license], Config), RawConfig = #{<<"key">> => LicenseKey}, emqx_config:put_raw([<<"license">>], RawConfig), - persistent_term:erase(emqx_license_test_pubkey), + emqx_license_test_lib:unmock_parser(), ok. set_special_configs(emqx_dashboard) -> @@ -48,10 +49,6 @@ set_special_configs(emqx_license) -> <<"connection_high_watermark">> => <<"80%">> }, emqx_config:put_raw([<<"license">>], RawConfig), - ok = persistent_term:put( - emqx_license_test_pubkey, - emqx_license_test_lib:public_key_pem() - ), ok; set_special_configs(_) -> ok. @@ -113,6 +110,19 @@ t_license_info(_Config) -> ), ok. +t_set_default_license(_Config) -> + NewKey = <<"default">>, + Res = request( + post, + uri(["license"]), + #{key => NewKey} + ), + ?assertMatch({ok, 200, _}, Res), + {ok, 200, Payload} = Res, + %% assert that it's not the string "default" returned + ?assertMatch(#{<<"customer">> := _}, emqx_utils_json:decode(Payload, [return_maps])), + ok. + t_license_upload_key_success(_Config) -> NewKey = emqx_license_test_lib:make_license(#{max_connections => "999"}), Res = request( @@ -194,6 +204,17 @@ t_license_setting(_Config) -> ?assertEqual(0.5, emqx_config:get([license, connection_low_watermark])), ?assertEqual(0.55, emqx_config:get([license, connection_high_watermark])), + %% update + Low1 = <<"50%">>, + High1 = <<"100%">>, + UpdateRes1 = request(put, uri(["license", "setting"]), #{ + <<"connection_low_watermark">> => Low1, + <<"connection_high_watermark">> => High1 + }), + validate_setting(UpdateRes1, Low1, High1), + ?assertEqual(0.5, emqx_config:get([license, connection_low_watermark])), + ?assertEqual(1.0, emqx_config:get([license, connection_high_watermark])), + %% update bad setting low >= high ?assertMatch( {ok, 400, _}, diff --git a/apps/emqx_license/test/emqx_license_parser_SUITE.erl b/apps/emqx_license/test/emqx_license_parser_SUITE.erl index f2f24dc54..0315a8a0b 100644 --- a/apps/emqx_license/test/emqx_license_parser_SUITE.erl +++ b/apps/emqx_license/test/emqx_license_parser_SUITE.erl @@ -40,6 +40,7 @@ set_special_configs(_) -> %%------------------------------------------------------------------------------ t_parse(_Config) -> + Parser = emqx_license_parser_v20220101, ?assertMatch({ok, _}, emqx_license_parser:parse(sample_license(), public_key_pem())), %% invalid version @@ -61,10 +62,7 @@ t_parse(_Config) -> ), ?assertMatch({error, _}, Res1), {error, Err1} = Res1, - ?assertEqual( - invalid_version, - proplists:get_value(emqx_license_parser_v20220101, Err1) - ), + ?assertMatch(#{error := invalid_version}, find_error(Parser, Err1)), %% invalid field number Res2 = emqx_license_parser:parse( @@ -87,9 +85,9 @@ t_parse(_Config) -> ), ?assertMatch({error, _}, Res2), {error, Err2} = Res2, - ?assertEqual( - unexpected_number_of_fields, - proplists:get_value(emqx_license_parser_v20220101, Err2) + ?assertMatch( + #{error := unexpected_number_of_fields}, + find_error(Parser, Err2) ), Res3 = emqx_license_parser:parse( @@ -110,14 +108,17 @@ t_parse(_Config) -> ), ?assertMatch({error, _}, Res3), {error, Err3} = Res3, - ?assertEqual( - [ - {type, invalid_license_type}, - {customer_type, invalid_customer_type}, - {date_start, invalid_date}, - {days, invalid_int_value} - ], - proplists:get_value(emqx_license_parser_v20220101, Err3) + ?assertMatch( + #{ + error := + #{ + type := invalid_license_type, + customer_type := invalid_customer_type, + date_start := invalid_date, + days := invalid_int_value + } + }, + find_error(Parser, Err3) ), Res4 = emqx_license_parser:parse( @@ -139,14 +140,17 @@ t_parse(_Config) -> ?assertMatch({error, _}, Res4), {error, Err4} = Res4, - ?assertEqual( - [ - {type, invalid_license_type}, - {customer_type, invalid_customer_type}, - {date_start, invalid_date}, - {days, invalid_int_value} - ], - proplists:get_value(emqx_license_parser_v20220101, Err4) + ?assertMatch( + #{ + error := + #{ + type := invalid_license_type, + customer_type := invalid_customer_type, + date_start := invalid_date, + days := invalid_int_value + } + }, + find_error(Parser, Err4) ), %% invalid signature @@ -189,14 +193,14 @@ t_parse(_Config) -> ), ?assertMatch({error, _}, Res5), {error, Err5} = Res5, - ?assertEqual( - invalid_signature, - proplists:get_value(emqx_license_parser_v20220101, Err5) + ?assertMatch( + #{error := invalid_signature}, + find_error(Parser, Err5) ), %% totally invalid strings as license ?assertMatch( - {error, [_ | _]}, + {error, #{parse_results := [#{error := bad_license_format}]}}, emqx_license_parser:parse( <<"badlicense">>, public_key_pem() @@ -204,7 +208,7 @@ t_parse(_Config) -> ), ?assertMatch( - {error, [_ | _]}, + {error, #{parse_results := [#{error := bad_license_format}]}}, emqx_license_parser:parse( <<"bad.license">>, public_key_pem() @@ -249,6 +253,20 @@ t_expiry_date(_Config) -> ?assertEqual({2295, 10, 27}, emqx_license_parser:expiry_date(License)). +t_empty_string(_Config) -> + ?assertMatch( + {error, #{ + parse_results := [ + #{ + error := empty_string, + module := emqx_license_parser_v20220101 + } + | _ + ] + }}, + emqx_license_parser:parse(<<>>) + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ @@ -270,3 +288,10 @@ sample_license() -> "10" ] ). + +find_error(Module, #{parse_results := Results}) -> + find_error(Module, Results); +find_error(Module, [#{module := Module} = Result | _Results]) -> + Result; +find_error(Module, [_Result | Results]) -> + find_error(Module, Results). diff --git a/apps/emqx_license/test/emqx_license_test_lib.erl b/apps/emqx_license/test/emqx_license_test_lib.erl index 644b6959d..c2f6c01e6 100644 --- a/apps/emqx_license/test/emqx_license_test_lib.erl +++ b/apps/emqx_license/test/emqx_license_test_lib.erl @@ -7,17 +7,6 @@ -compile(nowarn_export_all). -compile(export_all). --define(DEFAULT_LICENSE_VALUES, [ - "220111", - "0", - "10", - "Foo", - "contact@foo.com", - "20220111", - "100000", - "10" -]). - private_key() -> test_key("pvt.key"). @@ -76,5 +65,18 @@ make_license(Values) -> EncodedSignature = base64:encode(Signature), iolist_to_binary([EncodedText, ".", EncodedSignature]). +default_test_license() -> + make_license(#{}). + default_license() -> emqx_license_schema:default_license(). + +mock_parser() -> + meck:new(emqx_license_parser, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_license_parser, pubkey, fun() -> public_key_pem() end), + meck:expect(emqx_license_parser, default, fun() -> default_test_license() end), + ok. + +unmock_parser() -> + meck:unload(emqx_license_parser), + ok. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 496afcd64..6d7012313 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.16"}, + {vsn, "0.2.17"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index efa05ad37..f9deaf819 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.33"}, + {vsn, "5.0.34"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index d5879be36..d08bb9882 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -57,7 +57,7 @@ %% erlfmt-ignore -define(SYSMON_EXAMPLE, - <<""" + <<" sysmon { os { cpu_check_interval = 60s @@ -78,7 +78,7 @@ process_low_watermark = 60% } } - """>> + ">> ). api_spec() -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index c89ee202e..81a1103d2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -399,7 +399,7 @@ do_install_package(FileName, Bin) -> end, {400, #{ code => 'BAD_PLUGIN_INFO', - message => iolist_to_binary([Reason, ":", FileName]) + message => iolist_to_binary([Reason, ": ", FileName]) }} end. @@ -445,7 +445,8 @@ install_package(FileName, Bin) -> case emqx_plugins:ensure_installed(PackageName) of {error, #{return := not_found}} = NotFound -> NotFound; - {error, _Reason} = Error -> + {error, Reason} = Error -> + ?SLOG(error, Reason#{msg => "failed_to_install_plugin"}), _ = file:delete(File), Error; Result -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index f428009cb..32fbfdee5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -214,7 +214,22 @@ t_kickout_clients(_) -> {ok, C3} = emqtt:start_link(#{clientid => ClientId3}), {ok, _} = emqtt:connect(C3), - timer:sleep(300), + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + try + [_] = emqx_cm:lookup_channels(ClientId1), + [_] = emqx_cm:lookup_channels(ClientId2), + [_] = emqx_cm:lookup_channels(ClientId3), + true + catch + error:badmatch -> + false + end + end, + 2000 + ), %% get /clients ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), @@ -233,6 +248,15 @@ t_kickout_clients(_) -> KickoutBody = [ClientId1, ClientId2, ClientId3], {ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody), + ReceiveExit = fun({ClientPid, ClientId}) -> + receive + {'EXIT', Pid, _} when Pid =:= ClientPid -> + ok + after 1000 -> + error({timeout, ClientId}) + end + end, + lists:foreach(ReceiveExit, [{C1, ClientId1}, {C2, ClientId2}, {C3, ClientId3}]), {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]), ?assertMatch(#{<<"meta">> := #{<<"count">> := 0}}, ClientsResponse2). diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index c384b55e8..53d1fe589 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -464,6 +464,7 @@ apps_to_start() -> emqx_modules, emqx_gateway, emqx_exhook, + emqx_bridge_http, emqx_bridge, emqx_auto_subscribe, diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index e986a3fe1..377644cdf 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.23"}, + {vsn, "5.0.24"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 2212ac7d4..8279da934 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index 135f6878e..e9f7f6f98 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index 963d1ec39..b26836475 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 41538daf6..f14a1022a 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -83,7 +83,7 @@ describe(NameVsn) -> read_plugin(NameVsn, #{fill_readme => true}). %% @doc Install a .tar.gz package placed in install_dir. --spec ensure_installed(name_vsn()) -> ok | {error, any()}. +-spec ensure_installed(name_vsn()) -> ok | {error, map()}. ensure_installed(NameVsn) -> case read_plugin(NameVsn, #{}) of {ok, _} -> diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 3e9850129..b0a47a6a0 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -750,7 +750,7 @@ group_t_copy_plugin_to_a_new_node_single_node({init, Config}) -> | Config ]; group_t_copy_plugin_to_a_new_node_single_node({'end', Config}) -> - CopyToNode = proplists:get_value(copy_to_node, Config), + CopyToNode = proplists:get_value(copy_to_node_name, Config), ok = emqx_common_test_helpers:stop_slave(CopyToNode), ok = file:del_dir_r(proplists:get_value(to_install_dir, Config)), ok; diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src index efe422cd0..9c31b49c6 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.app.src +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -1,6 +1,6 @@ {application, emqx_postgresql, [ {description, "EMQX PostgreSQL Database Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl index 94e07ba7a..2b3f7febc 100644 --- a/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl +++ b/apps/emqx_postgresql/src/schema/emqx_postgresql_connector_schema.erl @@ -24,6 +24,7 @@ }). -export([ + namespace/0, roots/0, fields/1, desc/1 @@ -37,6 +38,9 @@ -define(CONNECTOR_TYPE, pgsql). +namespace() -> + "connector_postgres". + roots() -> []. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 4631fec8b..599e20fb7 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_redis/src/emqx_redis.app.src b/apps/emqx_redis/src/emqx_redis.app.src index e51c0fa80..1f8c5fbc3 100644 --- a/apps/emqx_redis/src/emqx_redis.app.src +++ b/apps/emqx_redis/src/emqx_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_redis, [ {description, "EMQX Redis Database Connector"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index 25d64e2fa..5435b3a9e 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -20,7 +20,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --export([roots/0, fields/1, redis_fields/0, desc/1]). +-export([namespace/0, roots/0, fields/1, redis_fields/0, desc/1]). -behaviour(emqx_resource). @@ -45,6 +45,8 @@ }). %%===================================================================== +namespace() -> "redis". + roots() -> [ {config, #{ diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 9edd03078..6649d0ef2 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.25"}, + {vsn, "0.1.26"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index df038a434..f67f1edb8 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1111,7 +1111,7 @@ is_channel_id(Id) -> %% Check if channel is installed in the connector state. %% There is no need to query the conncector if the channel is not %% installed as the query will fail anyway. -pre_query_channel_check({Id, _} = _Request, Channels) when +pre_query_channel_check({Id, _} = _Request, Channels, QueryOpts) when is_map_key(Id, Channels) -> ChannelStatus = maps:get(Id, Channels), @@ -1119,18 +1119,25 @@ pre_query_channel_check({Id, _} = _Request, Channels) when true -> ok; false -> - maybe_throw_channel_not_installed(Id) + maybe_throw_channel_not_installed(Id, QueryOpts) end; -pre_query_channel_check({Id, _} = _Request, _Channels) -> - maybe_throw_channel_not_installed(Id); -pre_query_channel_check(_Request, _Channels) -> +pre_query_channel_check({Id, _} = _Request, _Channels, QueryOpts) -> + maybe_throw_channel_not_installed(Id, QueryOpts); +pre_query_channel_check(_Request, _Channels, _QueryOpts) -> ok. -maybe_throw_channel_not_installed(Id) -> - %% Fail with a recoverable error if the channel is not installed - %% so that the operation can be retried. It is emqx_resource_manager's - %% responsibility to ensure that the channel installation is retried. +maybe_throw_channel_not_installed(Id, QueryOpts) -> + %% Fail with a recoverable error if the channel is not installed and there are buffer + %% workers involved so that the operation can be retried. Otherwise, this is + %% unrecoverable. It is emqx_resource_manager's responsibility to ensure that the + %% channel installation is retried. + IsSimpleQuery = maps:get(simple_query, QueryOpts, false), case is_channel_id(Id) of + true when IsSimpleQuery -> + error( + {unrecoverable_error, + iolist_to_binary(io_lib:format("channel: \"~s\" not operational", [Id]))} + ); true -> error( {recoverable_error, @@ -1191,7 +1198,7 @@ apply_query_fun( ?APPLY_RESOURCE( call_query, begin - pre_query_channel_check(Request, Channels), + pre_query_channel_check(Request, Channels, QueryOpts), Mod:on_query(extract_connector_id(Id), Request, ResSt) end, Request @@ -1222,7 +1229,7 @@ apply_query_fun( AsyncWorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Query, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), - pre_query_channel_check(Request, Channels), + pre_query_channel_check(Request, Channels, QueryOpts), Result = Mod:on_query_async( extract_connector_id(Id), Request, {ReplyFun, [ReplyContext]}, ResSt ), @@ -1249,7 +1256,7 @@ apply_query_fun( ?APPLY_RESOURCE( call_batch_query, begin - pre_query_channel_check(FirstRequest, Channels), + pre_query_channel_check(FirstRequest, Channels, QueryOpts), Mod:on_batch_query(extract_connector_id(Id), Requests, ResSt) end, Batch @@ -1291,7 +1298,7 @@ apply_query_fun( AsyncWorkerMRef = undefined, InflightItem = ?INFLIGHT_ITEM(Ref, Batch, IsRetriable, AsyncWorkerMRef), ok = inflight_append(InflightTID, InflightItem), - pre_query_channel_check(FirstRequest, Channels), + pre_query_channel_check(FirstRequest, Channels, QueryOpts), Result = Mod:on_batch_query_async( extract_connector_id(Id), Requests, {ReplyFun, [ReplyContext]}, ResSt ), diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 11391fb2b..67f22faee 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -1228,6 +1228,11 @@ channel_status({connecting, Error}) -> status => connecting, error => Error }; +channel_status(?status_disconnected) -> + #{ + status => ?status_disconnected, + error => <<"Disconnected for unknown reason">> + }; channel_status(connecting) -> #{ status => connecting, diff --git a/apps/emqx_resource/test/emqx_resource_schema_tests.erl b/apps/emqx_resource/test/emqx_resource_schema_tests.erl index 78a761bd2..b6cda8e97 100644 --- a/apps/emqx_resource/test/emqx_resource_schema_tests.erl +++ b/apps/emqx_resource/test/emqx_resource_schema_tests.erl @@ -80,7 +80,7 @@ worker_pool_size_test_() -> Conf = emqx_utils_maps:deep_put( [ <<"bridges">>, - <<"webhook">>, + <<"http">>, <<"simple">>, <<"resource_opts">>, <<"worker_pool_size">> @@ -88,7 +88,7 @@ worker_pool_size_test_() -> BaseConf, WorkerPoolSize ), - #{<<"bridges">> := #{<<"webhook">> := #{<<"simple">> := CheckedConf}}} = check(Conf), + #{<<"bridges">> := #{<<"http">> := #{<<"simple">> := CheckedConf}}} = check(Conf), #{<<"resource_opts">> := #{<<"worker_pool_size">> := WPS}} = CheckedConf, WPS end, @@ -117,7 +117,7 @@ worker_pool_size_test_() -> %%=========================================================================== parse_and_check_webhook_bridge(Hocon) -> - #{<<"bridges">> := #{<<"webhook">> := #{<<"simple">> := Conf}}} = check(parse(Hocon)), + #{<<"bridges">> := #{<<"http">> := #{<<"simple">> := Conf}}} = check(parse(Hocon)), Conf. parse(Hocon) -> @@ -134,7 +134,7 @@ check(Conf) when is_map(Conf) -> %% erlfmt-ignore webhook_bridge_health_check_hocon(HealthCheckInterval) -> io_lib:format( -""" +" bridges.webhook.simple { url = \"http://localhost:4000\" body = \"body\" @@ -142,5 +142,5 @@ bridges.webhook.simple { health_check_interval = \"~s\" } } -""", +", [HealthCheckInterval]). diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 7f5ceeff5..c90ba097b 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -30,7 +30,7 @@ {profiles, [ {test, [ {deps, [ - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.1"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.6"}}} ]} ]} ]}. diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index cab070826..6c0def7ae 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.18"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 8b27cdda4..d073d34e4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.29"}, + {vsn, "5.0.30"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index aadd3d4f5..afa57dfac 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -583,10 +583,18 @@ get_referenced_hookpoints(Froms) -> ]. get_egress_bridges(Actions) -> - [ - emqx_bridge_resource:bridge_id(BridgeType, BridgeName) - || {bridge, BridgeType, BridgeName, _ResId} <- Actions - ]. + lists:foldr( + fun + ({bridge, BridgeType, BridgeName, _ResId}, Acc) -> + [emqx_bridge_resource:bridge_id(BridgeType, BridgeName) | Acc]; + ({bridge_v2, BridgeType, BridgeName}, Acc) -> + [emqx_bridge_resource:bridge_id(BridgeType, BridgeName) | Acc]; + (_, Acc) -> + Acc + end, + [], + Actions + ). %% For allowing an external application to add extra "built-in" functions to the %% rule engine SQL like language. The module set by diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 14682eff1..f3df46b80 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -3468,7 +3468,7 @@ t_get_basic_usage_info_1(_Config) -> referenced_bridges => #{ mqtt => 1, - webhook => 3 + http => 3 } }, emqx_rule_engine:get_basic_usage_info() diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_schema_tests.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_schema_tests.erl index e3cff53e9..e361e2ad2 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_schema_tests.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_schema_tests.erl @@ -24,7 +24,7 @@ %% erlfmt-ignore republish_hocon0() -> -""" +" rule_engine.rules.my_rule { description = \"some desc\" metadata = {created_at = 1693918992079} @@ -55,7 +55,7 @@ rule_engine.rules.my_rule { } ] } -""". +". %%=========================================================================== %% Helper functions diff --git a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl index 07cb18e60..8f47e8290 100644 --- a/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl @@ -41,44 +41,32 @@ suite() -> apps() -> [ emqx_conf, - emqx_management, + emqx_connector, emqx_retainer, emqx_auth, emqx_auth_redis, emqx_auth_mnesia, emqx_auth_postgresql, emqx_modules, - emqx_telemetry + emqx_telemetry, + emqx_bridge_http, + emqx_bridge, + emqx_rule_engine, + emqx_management ]. init_per_suite(Config) -> - net_kernel:start(['master@127.0.0.1', longnames]), - ok = meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), - meck:expect( - emqx_authz_file, - acl_conf_file, - fun() -> - emqx_common_test_helpers:deps_path(emqx_auth, "etc/acl.conf") - end - ), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), - emqx_gateway_test_utils:load_all_gateway_apps(), - start_apps(), - Config. + WorkDir = ?config(priv_dir, Config), + Apps = emqx_cth_suite:start(apps(), #{work_dir => WorkDir}), + emqx_mgmt_api_test_util:init_suite(), + [{apps, Apps}, {work_dir, WorkDir} | Config]. -end_per_suite(_Config) -> - {ok, _} = emqx:update_config( - [authorization], - #{ - <<"no_match">> => <<"allow">>, - <<"cache">> => #{<<"enable">> => <<"true">>}, - <<"sources">> => [] - } - ), +end_per_suite(Config) -> mnesia:clear_table(cluster_rpc_commit), mnesia:clear_table(cluster_rpc_mfa), - stop_apps(), - meck:unload(emqx_authz_file), + Apps = ?config(apps, Config), + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_cth_suite:stop(Apps), ok. init_per_testcase(t_get_telemetry_without_memsup, Config) -> @@ -123,7 +111,6 @@ init_per_testcase(t_advanced_mqtt_features, Config) -> mock_advanced_mqtt_features(), Config; init_per_testcase(t_authn_authz_info, Config) -> - mock_httpc(), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), create_authn('mqtt:global', built_in_database), create_authn('tcp:default', redis), @@ -141,14 +128,11 @@ init_per_testcase(t_send_after_enable, Config) -> mock_httpc(), Config; init_per_testcase(t_rule_engine_and_data_bridge_info, Config) -> - mock_httpc(), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), - emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_bridge]), ok = emqx_bridge_SUITE:setup_fake_telemetry_data(), ok = setup_fake_rule_engine_data(), Config; init_per_testcase(t_exhook_info, Config) -> - mock_httpc(), {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), ExhookConf = #{ @@ -173,31 +157,8 @@ init_per_testcase(t_cluster_uuid, Config) -> Node = start_slave(n1), [{n1, Node} | Config]; init_per_testcase(t_uuid_restored_from_file, Config) -> - mock_httpc(), - NodeUUID = <<"AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE">>, - ClusterUUID = <<"FFFFFFFF-GGGG-HHHH-IIII-JJJJJJJJJJJJ">>, - DataDir = emqx:data_dir(), - NodeUUIDFile = filename:join(DataDir, "node.uuid"), - ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), - file:delete(NodeUUIDFile), - file:delete(ClusterUUIDFile), - ok = file:write_file(NodeUUIDFile, NodeUUID), - ok = file:write_file(ClusterUUIDFile, ClusterUUID), - - %% clear the UUIDs in the DB - {atomic, ok} = mria:clear_table(emqx_telemetry), - stop_apps(), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), - start_apps(), - Node = start_slave(n1), - [ - {n1, Node}, - {node_uuid, NodeUUID}, - {cluster_uuid, ClusterUUID} - | Config - ]; + Config; init_per_testcase(t_uuid_saved_to_file, Config) -> - mock_httpc(), DataDir = emqx:data_dir(), NodeUUIDFile = filename:join(DataDir, "node.uuid"), ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), @@ -205,7 +166,6 @@ init_per_testcase(t_uuid_saved_to_file, Config) -> file:delete(ClusterUUIDFile), Config; init_per_testcase(t_num_clients, Config) -> - mock_httpc(), ok = snabbkaffe:start_trace(), Config; init_per_testcase(_Testcase, Config) -> @@ -227,7 +187,6 @@ end_per_testcase(t_advanced_mqtt_features, _Config) -> {atomic, ok} = mria:clear_table(emqx_delayed), ok; end_per_testcase(t_authn_authz_info, _Config) -> - meck:unload([httpc]), emqx_authz:update({delete, postgresql}, #{}), lists:foreach( fun(ChainName) -> @@ -244,19 +203,8 @@ end_per_testcase(t_enable, _Config) -> end_per_testcase(t_send_after_enable, _Config) -> meck:unload([httpc, emqx_telemetry_config]); end_per_testcase(t_rule_engine_and_data_bridge_info, _Config) -> - meck:unload(httpc), - lists:foreach( - fun(App) -> - ok = application:stop(App) - end, - [ - emqx_bridge, - emqx_rule_engine - ] - ), ok; end_per_testcase(t_exhook_info, _Config) -> - meck:unload(httpc), emqx_exhook_demo_svr:stop(), application:stop(emqx_exhook), ok; @@ -264,21 +212,12 @@ end_per_testcase(t_cluster_uuid, Config) -> Node = proplists:get_value(n1, Config), ok = stop_slave(Node); end_per_testcase(t_num_clients, Config) -> - meck:unload([httpc]), ok = snabbkaffe:stop(), Config; -end_per_testcase(t_uuid_restored_from_file, Config) -> - Node = ?config(n1, Config), - DataDir = emqx:data_dir(), - NodeUUIDFile = filename:join(DataDir, "node.uuid"), - ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), - ok = file:delete(NodeUUIDFile), - ok = file:delete(ClusterUUIDFile), - meck:unload([httpc]), - ok = stop_slave(Node), - ok; end_per_testcase(_Testcase, _Config) -> - meck:unload([httpc]), + case catch meck:unload([httpc]) of + _ -> ok + end, ok. %%------------------------------------------------------------------------------ @@ -315,19 +254,34 @@ t_cluster_uuid(Config) -> %% should attempt read UUID from file in data dir to keep UUIDs %% unique, in the event of a database purge. t_uuid_restored_from_file(Config) -> - ExpectedNodeUUID = ?config(node_uuid, Config), - ExpectedClusterUUID = ?config(cluster_uuid, Config), + %% Stop the emqx_telemetry application first + {atomic, ok} = mria:clear_table(emqx_telemetry), + application:stop(emqx_telemetry), + + %% Rewrite the the uuid files + NodeUUID = <<"AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE">>, + ClusterUUID = <<"FFFFFFFF-GGGG-HHHH-IIII-JJJJJJJJJJJJ">>, + DataDir = ?config(work_dir, Config), + NodeUUIDFile = filename:join(DataDir, "node.uuid"), + ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), + ok = file:write_file(NodeUUIDFile, NodeUUID), + ok = file:write_file(ClusterUUIDFile, ClusterUUID), + + %% Start the emqx_telemetry application again + application:start(emqx_telemetry), + + %% Check the UUIDs ?assertEqual( - {ok, ExpectedNodeUUID}, + {ok, NodeUUID}, emqx_telemetry:get_node_uuid() ), ?assertEqual( - {ok, ExpectedClusterUUID}, + {ok, ClusterUUID}, emqx_telemetry:get_cluster_uuid() ), ok. -t_uuid_saved_to_file(_Config) -> +t_uuid_saved_to_file(Config) -> DataDir = emqx:data_dir(), NodeUUIDFile = filename:join(DataDir, "node.uuid"), ClusterUUIDFile = filename:join(DataDir, "cluster.uuid"), @@ -337,9 +291,10 @@ t_uuid_saved_to_file(_Config) -> %% clear the UUIDs in the DB {atomic, ok} = mria:clear_table(emqx_telemetry), - stop_apps(), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF), - start_apps(), + application:stop(emqx_telemetry), + + application:start(emqx_telemetry), + {ok, NodeUUID} = emqx_telemetry:get_node_uuid(), {ok, ClusterUUID} = emqx_telemetry:get_cluster_uuid(), ?assertEqual( @@ -578,6 +533,7 @@ t_mqtt_runtime_insights(_) -> t_rule_engine_and_data_bridge_info(_Config) -> {ok, TelemetryData} = emqx_telemetry:get_telemetry(), + ct:pal("telemetry data: ~p~n", [TelemetryData]), RuleInfo = get_value(rule_engine, TelemetryData), BridgeInfo = get_value(bridge, TelemetryData), ?assertEqual( @@ -588,7 +544,7 @@ t_rule_engine_and_data_bridge_info(_Config) -> #{ data_bridge => #{ - webhook => #{num => 1, num_linked_by_rules => 3}, + http => #{num => 1, num_linked_by_rules => 3}, mqtt => #{num => 2, num_linked_by_rules => 2} }, num_data_bridges => 3 @@ -811,14 +767,6 @@ setup_fake_rule_engine_data() -> ), ok. -set_special_configs(emqx_auth) -> - {ok, _} = emqx:update_config([authorization, cache, enable], false), - {ok, _} = emqx:update_config([authorization, no_match], deny), - {ok, _} = emqx:update_config([authorization, sources], []), - ok; -set_special_configs(_App) -> - ok. - %% for some unknown reason, gen_rpc running locally or in CI might %% start with different `port_discovery' modes, which means that'll %% either be listening at the port in the config (`tcp_server_port', @@ -869,7 +817,7 @@ stop_slave(Node) -> % This line don't work!! %emqx_cluster_rpc:fast_forward_to_commit(Node, 100), rpc:call(Node, ?MODULE, leave_cluster, []), - ok = slave:stop(Node), + ok = emqx_cth_peer:stop(Node), ?assertEqual([node()], mria:running_nodes()), ?assertEqual([], nodes()), _ = application:stop(mria), @@ -887,9 +835,3 @@ leave_cluster() -> is_official_version(V) -> emqx_telemetry_config:is_official_version(V). - -start_apps() -> - emqx_common_test_helpers:start_apps(apps(), fun set_special_configs/1). - -stop_apps() -> - emqx_common_test_helpers:stop_apps(lists:reverse(apps())). diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index a86a8d841..c666d2069 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/changes/e5.3.2.en.md b/changes/e5.3.2.en.md new file mode 100644 index 000000000..07a924b38 --- /dev/null +++ b/changes/e5.3.2.en.md @@ -0,0 +1,42 @@ +# e5.3.2 + +## Enhancements + +- [#11752](https://github.com/emqx/emqx/pull/11752) Changed default RPC driver from `gen_rpc` to `rpc` for core-replica database synchronization. + + This improves core-replica data replication latency. + +- [#11785](https://github.com/emqx/emqx/pull/11785) Allowed users with the "Viewer" role to change their own passwords. However, those with the "Viewer" role do not have permission to change the passwords of other users. + +- [#11787](https://github.com/emqx/emqx/pull/11787) Improved the performance of the `emqx` command. + +- [#11790](https://github.com/emqx/emqx/pull/11790) Added validation to Redis commands in Redis authorization source. + Additionally, this improvement refines the parsing of Redis commands during authentication and authorization processes. The parsing now aligns with `redis-cli` compatibility standards and supports quoted arguments. + +- [#11541](https://github.com/emqx/emqx/pull/11541) Enhanced file transfer capabilities. Now, clients can use an asynchronous method for file transfer by sending commands to the `$file-async/...` topic and subscribing to command execution results on the `$file-response/{clientId}` topic. This improvement simplifies the use of the file transfer feature, particularly suitable for clients using MQTT v3.1/v3.1.1 or those employing MQTT bridging. For more details, please refer to [EIP-0021](https://github.com/emqx/eip). + +## Bug Fixes + +- [#11757](https://github.com/emqx/emqx/pull/11757) Fixed the error response code when downloading non-existent trace files. Now the response returns `404` instead of `500`. + +- [#11762](https://github.com/emqx/emqx/pull/11762) Fixed an issue in EMQX's `built_in_database` authorization source. With this update, all Access Control List (ACL) records are completely removed when an authorization source is deleted. This resolves the issue of residual records remaining in the database when re-creating authorization sources. + +- [#11771](https://github.com/emqx/emqx/pull/11771) Fixed validation of Bcrypt salt rounds in authentication management through the API/Dashboard. + +- [#11780](https://github.com/emqx/emqx/pull/11780) Fixed validation of the `iterations` field of the `pbkdf2` password hashing algorithm. Now, `iterations` must be strictly positive. Previously, it could be set to 0, which led to a nonfunctional authenticator. + +- [#11791](https://github.com/emqx/emqx/pull/11791) Fixed an issue in the EMQX CoAP Gateway where heartbeats were not effectively maintaining the connection's active status. This fix ensures that the heartbeat mechanism properly sustains the liveliness of CoAP Gateway connections. + +- [#11797](https://github.com/emqx/emqx/pull/11797) Modified HTTP API behavior for APIs managing the `built_in_database` authorization source. They will now return a `404` status code if `built_in_database` is not set as the authorization source, replacing the former `20X` response. + +- [#11965](https://github.com/emqx/emqx/pull/11965) Improved the termination of EMQX services to ensure a graceful stop even in the presence of an unavailable MongoDB resource. + +- [#11975](https://github.com/emqx/emqx/pull/11975) This fix addresses an issue where redundant error logs were generated due to a race condition during simultaneous socket closure by a peer and the server. Previously, concurrent socket close events triggered by the operating system and EMQX resulted in unnecessary error logging. The implemented fix improves event handling to eliminate unnecessary error messages. + +- [#11987](https://github.com/emqx/emqx/pull/11987) Fixed a bug where attempting to set the `active_n` option on a TCP/SSL socket could lead to a connection crash. + + The problem occurred if the socket had already been closed by the time the connection process attempted to apply the `active_n` setting, resulting in a `case_clause` crash. + +- [#11731](https://github.com/emqx/emqx/pull/11731) Added hot configuration support for the file transfer feature. + +- [#11754](https://github.com/emqx/emqx/pull/11754) Improved the log formatting specifically for the Postgres bridge in EMQX. It addresses issues related to Unicode characters in error messages returned by the driver. diff --git a/changes/ee/feat-12016.en.md b/changes/ee/feat-12016.en.md new file mode 100644 index 000000000..36c5115f6 --- /dev/null +++ b/changes/ee/feat-12016.en.md @@ -0,0 +1,4 @@ +Enhanced license key management. + +EMQX can now load the license key from a specified file. This is enabled by setting the `license.key` configuration to a file path, which should be prefixed with `"file://"`. +Also added the ability to revert to the default trial license by setting `license.key = default`. This option simplifies the process of returning to the trial license if needed. diff --git a/changes/v5.3.2.en.md b/changes/v5.3.2.en.md new file mode 100644 index 000000000..dc8f7bdcc --- /dev/null +++ b/changes/v5.3.2.en.md @@ -0,0 +1,38 @@ +# v5.3.2 + +## Enhancements + +- [#11725](https://github.com/emqx/emqx/pull/11725) Introduced the LDAP as a new authentication and authorization backend. + +- [#11752](https://github.com/emqx/emqx/pull/11752) Changed default RPC driver from `gen_rpc` to `rpc` for core-replica database synchronization. + + This improves core-replica data replication latency. + +- [#11785](https://github.com/emqx/emqx/pull/11785) Allowed users with the "Viewer" role to change their own passwords. However, those with the "Viewer" role do not have permission to change the passwords of other users. + +- [#11787](https://github.com/emqx/emqx/pull/11787) Improved the performance of the `emqx` command. + +- [#11790](https://github.com/emqx/emqx/pull/11790) Added validation to Redis commands in Redis authorization source. + Additionally, this improvement refines the parsing of Redis commands during authentication and authorization processes. The parsing now aligns with `redis-cli` compatibility standards and supports quoted arguments. + +## Bug Fixes + +- [#11757](https://github.com/emqx/emqx/pull/11757) Fixed the error response code when downloading non-existent trace files. Now the response returns `404` instead of `500`. + +- [#11762](https://github.com/emqx/emqx/pull/11762) Fixed an issue in EMQX's `built_in_database` authorization source. With this update, all Access Control List (ACL) records are completely removed when an authorization source is deleted. This resolves the issue of residual records remaining in the database when re-creating authorization sources. + +- [#11771](https://github.com/emqx/emqx/pull/11771) Fixed validation of Bcrypt salt rounds in authentication management through the API/Dashboard. + +- [#11780](https://github.com/emqx/emqx/pull/11780) Fixed validation of the `iterations` field of the `pbkdf2` password hashing algorithm. Now, `iterations` must be strictly positive. Previously, it could be set to 0, which led to a nonfunctional authenticator. + +- [#11791](https://github.com/emqx/emqx/pull/11791) Fixed an issue in the EMQX CoAP Gateway where heartbeats were not effectively maintaining the connection's active status. This fix ensures that the heartbeat mechanism properly sustains the liveliness of CoAP Gateway connections. + +- [#11797](https://github.com/emqx/emqx/pull/11797) Modified HTTP API behavior for APIs managing the `built_in_database` authorization source. They will now return a `404` status code if `built_in_database` is not set as the authorization source, replacing the former `20X` response. + +- [#11965](https://github.com/emqx/emqx/pull/11965) Improved the termination of EMQX services to ensure a graceful stop even in the presence of an unavailable MongoDB resource. + +- [#11975](https://github.com/emqx/emqx/pull/11975) This fix addresses an issue where redundant error logs were generated due to a race condition during simultaneous socket closure by a peer and the server. Previously, concurrent socket close events triggered by the operating system and EMQX resulted in unnecessary error logging. The implemented fix improves event handling to eliminate unnecessary error messages. + +- [#11987](https://github.com/emqx/emqx/pull/11987) Fixed a bug where attempting to set the `active_n` option on a TCP/SSL socket could lead to a connection crash. + + The problem occurred if the socket had already been closed by the time the connection process attempted to apply the `active_n` setting, resulting in a `case_clause` crash. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index aed38cd63..652b2bcf5 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.3.2-alpha.1 +version: 5.3.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.2-alpha.1 +appVersion: 5.3.2 diff --git a/mix.exs b/mix.exs index 123273266..cd0424155 100644 --- a/mix.exs +++ b/mix.exs @@ -46,17 +46,17 @@ defmodule EMQXUmbrella.MixProject do # other exact versions, and not ranges. [ {:lc, github: "emqx/lc", tag: "0.3.2", override: true}, - {:redbug, "2.0.8"}, + {:redbug, github: "emqx/redbug", tag: "2.0.10"}, {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.11", override: true}, {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, - {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.8", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, - {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.1", override: true}, + {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.2", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.14", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, @@ -230,7 +230,7 @@ defmodule EMQXUmbrella.MixProject do {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.8.0"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, - {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"}, + {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.9", override: true}, {:crc32cer, "0.1.8", override: true}, @@ -823,7 +823,7 @@ defmodule EMQXUmbrella.MixProject do defp jq_dep() do if enable_jq?(), - do: [{:jq, github: "emqx/jq", tag: "v0.3.11", override: true}], + do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index efdba5eaa..5b05f25e9 100644 --- a/rebar.config +++ b/rebar.config @@ -51,7 +51,7 @@ {deps, [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} - , {redbug, "2.0.8"} + , {redbug, {git, "https://github.com/emqx/redbug", {tag, "2.0.10"}}} , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} , {gpb, "4.19.9"} , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} @@ -60,10 +60,10 @@ , {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} - , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.1"}}} + , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.14"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} diff --git a/rebar.config.erl b/rebar.config.erl index c3ace671c..e16be99d8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -16,7 +16,7 @@ do(Dir, CONFIG) -> assert_otp() -> Oldest = 24, - Latest = 25, + Latest = 26, OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < Oldest orelse OtpRelease > Latest of true -> @@ -42,7 +42,7 @@ quicer() -> {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.303"}}}. jq() -> - {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.11"}}}. + {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.12"}}}. deps(Config) -> {deps, OldDeps} = lists:keyfind(deps, 1, Config), @@ -53,7 +53,10 @@ deps(Config) -> lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}). overrides() -> - [{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}] ++ snabbkaffe_overrides(). + [ + {add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}, + {add, jesse, [{erl_opts, [nowarn_match_float_zero]}]} + ] ++ snabbkaffe_overrides(). %% Temporary workaround for a rebar3 erl_opts duplication %% bug. Ideally, we want to set this define globally diff --git a/rel/i18n/emqx_bridge_http_schema.hocon b/rel/i18n/emqx_bridge_http_schema.hocon index b7b715db1..416f77834 100644 --- a/rel/i18n/emqx_bridge_http_schema.hocon +++ b/rel/i18n/emqx_bridge_http_schema.hocon @@ -18,10 +18,10 @@ config_direction.desc: config_direction.label: """Bridge Direction""" -config_enable.desc: +config_enable_bridge.desc: """Enable or disable this bridge""" -config_enable.label: +config_enable_bridge.label: """Enable Or Disable Bridge""" config_headers.desc: @@ -71,6 +71,21 @@ is not allowed.""" config_url.label: """HTTP Bridge""" +config_path.desc: +"""The URL path for this Action.
+This path will be appended to the Connector's url configuration to form the full +URL address. +Template with variables is allowed in this option. For example, /room/{$room_no}""" + +config_path.label: +"""URL Path""" + +config_parameters_opts.desc: +"""The parameters for HTTP action.""" + +config_parameters_opts.label: +"""Parameters""" + desc_config.desc: """Configuration for an HTTP bridge.""" diff --git a/rel/i18n/emqx_license_schema.hocon b/rel/i18n/emqx_license_schema.hocon index 51387ed39..e280af257 100644 --- a/rel/i18n/emqx_license_schema.hocon +++ b/rel/i18n/emqx_license_schema.hocon @@ -25,7 +25,16 @@ connection_low_watermark_field_deprecated.label: """deprecated use /license/setting instead""" key_field.desc: -"""License string""" +"""This configuration parameter is designated for the license key and supports below input formats: + +- Direct Key: Enter the secret key directly as a string value. +- File Path: Specify the path to a file that contains the secret key. Ensure the path starts with file://. +- "default": Use string value "default" to apply the default trial license. + +Note: An invalid license key or an incorrect file path may prevent EMQX from starting successfully. +If a file path is used, EMQX attempts to reload the license key from the file every 2 minutes. +Any failure in reloading the license file will be recorded as an error level log message, +and EMQX continues to apply the license loaded previously.""" key_field.label: """License string""" diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index d931c66b1..2a6fb03ba 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1571,4 +1571,10 @@ session_builtin_n_shards.desc: session_storage_backend_builtin.desc: """Builtin session storage backend utilizing embedded RocksDB key-value store.""" +session_ds_session_gc_interval.desc: +"""The interval at which session garbage collection is executed for persistent sessions.""" + +session_ds_session_gc_batch_size.desc: +"""The size of each batch of expired persistent sessions to be garbage collected per iteration.""" + } diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 12c492132..054deabd4 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -18,6 +18,9 @@ case ${OTP_VSN} in 25*) VERSION="3.19.0-emqx-9" ;; + 26*) + VERSION="3.20.0-emqx-1" + ;; *) echo "Unsupporetd Erlang/OTP version $OTP_VSN" exit 1