From f0ab8e36d172543a206734fb658b5d35ece261ff Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Jul 2023 13:39:33 +0200 Subject: [PATCH 01/28] fix(machine): Don't start business apps before emqx machine --- mix.exs | 205 ++++++++++++++++++++++++++--------------------- rebar.config.erl | 82 ++++++++++--------- 2 files changed, 161 insertions(+), 126 deletions(-) diff --git a/mix.exs b/mix.exs index 9305b2d57..0872a3311 100644 --- a/mix.exs +++ b/mix.exs @@ -326,7 +326,7 @@ defmodule EMQXUmbrella.MixProject do end def applications(edition_type) do - [ + system_apps = [ crypto: :permanent, public_key: :permanent, asn1: :permanent, @@ -339,96 +339,121 @@ defmodule EMQXUmbrella.MixProject do redbug: :permanent, xmerl: :permanent, hocon: :load, - telemetry: :permanent, - emqx: :load, - emqx_conf: :load, - emqx_machine: :permanent - ] ++ - if(enable_rocksdb?(), - do: [mnesia_rocksdb: :load], - else: [] - ) ++ - [ - mnesia: :load, - ekka: :load, - esasl: :load, - observer_cli: :permanent, - tools: :permanent, - covertool: :load, - system_monitor: :load, - emqx_utils: :load, - emqx_http_lib: :permanent, - emqx_resource: :permanent, - emqx_connector: :permanent, - emqx_authn: :permanent, - emqx_authz: :permanent, - emqx_auto_subscribe: :permanent, - emqx_gateway: :permanent, - emqx_gateway_stomp: :permanent, - emqx_gateway_mqttsn: :permanent, - emqx_gateway_coap: :permanent, - emqx_gateway_lwm2m: :permanent, - emqx_gateway_exproto: :permanent, - emqx_exhook: :permanent, - emqx_bridge: :permanent, - emqx_bridge_mqtt: :permanent, - emqx_bridge_http: :permanent, - emqx_rule_engine: :permanent, - emqx_modules: :permanent, - emqx_management: :permanent, - emqx_dashboard: :permanent, - emqx_retainer: :permanent, - emqx_prometheus: :permanent, - emqx_psk: :permanent, - emqx_slow_subs: :permanent, - emqx_mongodb: :permanent, - emqx_redis: :permanent, - emqx_mysql: :permanent, - emqx_plugins: :permanent, - emqx_mix: :none - ] ++ - if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ - if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ - if(enable_jq?(), do: [jq: :load], else: []) ++ - if(is_app(:observer), - do: [observer: :load], - else: [] - ) ++ - if(edition_type == :enterprise, - do: [ - emqx_license: :permanent, - emqx_enterprise: :load, - emqx_bridge_kafka: :permanent, - emqx_bridge_pulsar: :permanent, - emqx_bridge_gcp_pubsub: :permanent, - emqx_bridge_cassandra: :permanent, - emqx_bridge_opents: :permanent, - emqx_bridge_clickhouse: :permanent, - emqx_bridge_dynamo: :permanent, - emqx_bridge_hstreamdb: :permanent, - emqx_bridge_influxdb: :permanent, - emqx_bridge_iotdb: :permanent, - emqx_bridge_matrix: :permanent, - emqx_bridge_mongodb: :permanent, - emqx_bridge_mysql: :permanent, - emqx_bridge_pgsql: :permanent, - emqx_bridge_redis: :permanent, - emqx_bridge_rocketmq: :permanent, - emqx_bridge_tdengine: :permanent, - emqx_bridge_timescale: :permanent, - emqx_bridge_sqlserver: :permanent, - emqx_oracle: :permanent, - emqx_bridge_oracle: :permanent, - emqx_bridge_rabbitmq: :permanent, - emqx_schema_registry: :permanent, - emqx_eviction_agent: :permanent, - emqx_node_rebalance: :permanent, - emqx_ft: :permanent - ], - else: [ - emqx_telemetry: :permanent + telemetry: :permanent + ] + + db_apps = + if enable_rocksdb?() do + [:mnesia_rocksdb] + else + [] + end ++ + [ + :mnesia, + :mria, + :ekka ] - ) + + business_apps = + [ + :emqx, + :emqx_conf, + :esasl, + :observer_cli, + :tools, + :covertool, + :system_monitor, + :emqx_utils, + :emqx_http_lib, + :emqx_resource, + :emqx_connector, + :emqx_authn, + :emqx_authz, + :emqx_auto_subscribe, + :emqx_gateway, + :emqx_gateway_stomp, + :emqx_gateway_mqttsn, + :emqx_gateway_coap, + :emqx_gateway_lwm2m, + :emqx_gateway_exproto, + :emqx_exhook, + :emqx_bridge, + :emqx_bridge_mqtt, + :emqx_bridge_http, + :emqx_rule_engine, + :emqx_modules, + :emqx_management, + :emqx_dashboard, + :emqx_retainer, + :emqx_prometheus, + :emqx_psk, + :emqx_slow_subs, + :emqx_mongodb, + :emqx_redis, + :emqx_mysql, + :emqx_plugins, + :emqx_mix + ] ++ + if enable_quicer?() do + [:quicer] + else + [] + end ++ + if enable_bcrypt?() do + [:bcrypt] + else + [] + end ++ + if enable_jq?() do + [:jq] + else + [] + end ++ + if(is_app(:observer), + do: [:observer], + else: [] + ) ++ + case edition_type do + :enterprise -> + [ + :emqx_license, + :emqx_enterprise, + :emqx_bridge_kafka, + :emqx_bridge_pulsar, + :emqx_bridge_gcp_pubsub, + :emqx_bridge_cassandra, + :emqx_bridge_opents, + :emqx_bridge_clickhouse, + :emqx_bridge_dynamo, + :emqx_bridge_hstreamdb, + :emqx_bridge_influxdb, + :emqx_bridge_iotdb, + :emqx_bridge_matrix, + :emqx_bridge_mongodb, + :emqx_bridge_mysql, + :emqx_bridge_pgsql, + :emqx_bridge_redis, + :emqx_bridge_rocketmq, + :emqx_bridge_tdengine, + :emqx_bridge_timescale, + :emqx_bridge_sqlserver, + :emqx_oracle, + :emqx_bridge_oracle, + :emqx_bridge_rabbitmq, + :emqx_schema_registry, + :emqx_eviction_agent, + :emqx_node_rebalance, + :emqx_ft + ] + + _ -> + [:emqx_telemetry] + end + + system_apps ++ + Enum.map(db_apps, &{&1, :load}) ++ + [emqx_machine: :permanent] ++ + Enum.map(business_apps, &{&1, :load}) end defp is_app(name) do diff --git a/rebar.config.erl b/rebar.config.erl index 5a3ec1355..f33d84006 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -386,38 +386,43 @@ overlay_vars_pkg(pkg) -> ]. relx_apps(ReleaseType, Edition) -> - [ - kernel, - sasl, - crypto, - public_key, - asn1, - syntax_tools, - ssl, - os_mon, - inets, - compiler, - runtime_tools, - redbug, - xmerl, - {hocon, load}, - telemetry, - % started by emqx_machine - {emqx, load}, - {emqx_conf, load}, - emqx_machine - ] ++ - [{mnesia_rocksdb, load} || is_rocksdb_supported()] ++ + SystemApps = [ - {mnesia, load}, - {ekka, load}, - {esasl, load}, + kernel, + sasl, + crypto, + public_key, + asn1, + syntax_tools, + ssl, + os_mon, + inets, + compiler, + runtime_tools, + redbug, + xmerl, + {hocon, load}, + telemetry + ], + DBApps = + [mnesia_rocksdb || is_rocksdb_supported()] ++ + [ + mnesia, + mria, + ekka + ], + BusinessApps = + [ + emqx, + emqx_conf, + + esasl, observer_cli, tools, - {covertool, load}, + covertool, % started by emqx_machine - {system_monitor, load}, - {emqx_utils, load}, + system_monitor, + emqx_utils, emqx_http_lib, emqx_resource, emqx_connector, @@ -447,13 +452,18 @@ relx_apps(ReleaseType, Edition) -> emqx_mysql, emqx_plugins ] ++ - [quicer || is_quicer_supported()] ++ - [bcrypt || provide_bcrypt_release(ReleaseType)] ++ - %% Started automatically when needed (only needs to be started when the - %% port implementation is used) - [{jq, load} || is_jq_supported()] ++ - [{observer, load} || is_app(observer)] ++ - relx_apps_per_edition(Edition). + [quicer || is_quicer_supported()] ++ + [bcrypt || provide_bcrypt_release(ReleaseType)] ++ + %% Started automatically when needed (only needs to be started when the + %% port implementation is used) + [jq || is_jq_supported()] ++ + [observer || is_app(observer)] ++ + relx_apps_per_edition(Edition), + SystemApps ++ + %% EMQX starts the DB and the business applications: + [{App, load} || App <- DBApps] ++ + [emqx_machine] ++ + [{App, load} || App <- BusinessApps]. is_app(Name) -> case application:load(Name) of @@ -465,7 +475,7 @@ is_app(Name) -> relx_apps_per_edition(ee) -> [ emqx_license, - {emqx_enterprise, load}, + emqx_enterprise, emqx_bridge_kafka, emqx_bridge_pulsar, emqx_bridge_gcp_pubsub, From 293700773fb26b830f3443d227a603d1a5fe7147 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:30:19 +0200 Subject: [PATCH 02/28] refactor(mgmt_cli): Call mria directly --- apps/emqx_management/src/emqx_mgmt_cli.erl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index e0685b2ff..fcf14f17a 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -110,7 +110,7 @@ broker(_) -> %% @doc Cluster with other nodes cluster(["join", SNode]) -> - case ekka:join(ekka_node:parse_name(SNode)) of + case mria:join(ekka_node:parse_name(SNode)) of ok -> emqx_ctl:print("Join the cluster successfully.~n"), cluster(["status"]); @@ -120,7 +120,7 @@ cluster(["join", SNode]) -> emqx_ctl:print("Failed to join the cluster: ~0p~n", [Error]) end; cluster(["leave"]) -> - case ekka:leave() of + case mria:leave() of ok -> emqx_ctl:print("Leave the cluster successfully.~n"), cluster(["status"]); @@ -128,7 +128,7 @@ cluster(["leave"]) -> emqx_ctl:print("Failed to leave the cluster: ~0p~n", [Error]) end; cluster(["force-leave", SNode]) -> - case ekka:force_leave(ekka_node:parse_name(SNode)) of + case mria:force_leave(ekka_node:parse_name(SNode)) of ok -> emqx_ctl:print("Remove the node from cluster successfully.~n"), cluster(["status"]); @@ -138,9 +138,9 @@ cluster(["force-leave", SNode]) -> emqx_ctl:print("Failed to remove the node from cluster: ~0p~n", [Error]) end; cluster(["status"]) -> - emqx_ctl:print("Cluster status: ~p~n", [ekka_cluster:info()]); + emqx_ctl:print("Cluster status: ~p~n", [cluster_info()]); cluster(["status", "--json"]) -> - Info = sort_map_list_fields(ekka_cluster:info()), + Info = sort_map_list_fields(cluster_info()), emqx_ctl:print("~ts~n", [emqx_logger_jsonfmt:best_effort_json(Info)]); cluster(_) -> emqx_ctl:usage([ @@ -925,3 +925,8 @@ with_log(Fun, Msg) -> {error, Reason} -> emqx_ctl:print("~s FAILED~n~p~n", [Msg, Reason]) end. + +cluster_info() -> + #{ running_nodes => mria:running_nodes() + , stopped_nodes => mria:cluster_nodes(stopped) + }. From 103b89f60ce232ec8422355d41d3a8552b7a6c02 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:37:57 +0200 Subject: [PATCH 03/28] fix(mgmt_cli): Don't crash when mria is not running --- apps/emqx_management/src/emqx_mgmt_cli.erl | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index fcf14f17a..f778ec790 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -927,6 +927,24 @@ with_log(Fun, Msg) -> end. cluster_info() -> - #{ running_nodes => mria:running_nodes() - , stopped_nodes => mria:cluster_nodes(stopped) - }. + RunningNodes = safe_call_mria(running_nodes, [], []), + StoppedNodes = safe_call_mria(cluster_nodes, [stopped], []), + #{ + running_nodes => RunningNodes, + stopped_nodes => StoppedNodes + }. + +%% CLI starts before mria, so we should handle errors gracefully: +safe_call_mria(Fun, Args, OnFail) -> + try + apply(mria, Fun, Args) + catch + EC:Err:Stack -> + ?SLOG(warning, #{ + msg => "Call to mria failed", + call => {mria, Fun, Args}, + EC => Err, + stacktrace => Stack + }), + OnFail + end. From ce76d4ab04ab0bd20b5dd1f3a24a4c3df082fa5d Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:57:19 +0200 Subject: [PATCH 04/28] chore(ekka): Bump version to 0.15.7 (mria 0.5.9) --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 78a8b76e0..2b6e8f172 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.5"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, diff --git a/mix.exs b/mix.exs index 0872a3311..fa2b291d8 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-11", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.5", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.7", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.11", override: true}, diff --git a/rebar.config b/rebar.config index 0f6864c5e..287b7b0b6 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-11"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.5"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.11"}}} From 3a5112829c84a0d7902ae394dbe864c4fd854769 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:07:10 +0200 Subject: [PATCH 05/28] fix(mgmt_cli): Don't print cluster status on the replicant --- apps/emqx_management/src/emqx_mgmt_cli.erl | 10 ++++++---- changes/ce/fix-11294.en.md | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-11294.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index f778ec790..9692441a6 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -113,7 +113,11 @@ cluster(["join", SNode]) -> case mria:join(ekka_node:parse_name(SNode)) of ok -> emqx_ctl:print("Join the cluster successfully.~n"), - cluster(["status"]); + %% FIXME: running status on the replicant immediately + %% after join produces stale output + mria_rlog:role() =:= core andalso + cluster(["status"]), + ok; ignore -> emqx_ctl:print("Ignore.~n"); {error, Error} -> @@ -158,9 +162,7 @@ sort_map_list_fields(Map) when is_map(Map) -> end, Map, maps:keys(Map) - ); -sort_map_list_fields(NotMap) -> - NotMap. + ). sort_map_list_field(Field, Map) -> case maps:get(Field, Map) of diff --git a/changes/ce/fix-11294.en.md b/changes/ce/fix-11294.en.md new file mode 100644 index 000000000..c9ee1cdc5 --- /dev/null +++ b/changes/ce/fix-11294.en.md @@ -0,0 +1 @@ +Fix `emqx_ctl cluster join`, `leave`, and `status` commands. From 59ed65787f30f1119e9ff2f4b24dce5846a7a1d5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jul 2023 18:03:50 +0800 Subject: [PATCH 06/28] fix: get_config return target node's node conf --- apps/emqx_management/src/emqx_mgmt.erl | 16 ++-- .../src/emqx_mgmt_api_configs.erl | 17 ++-- .../src/emqx_mgmt_api_listeners.erl | 2 +- .../src/proto/emqx_management_proto_v5.erl | 96 +++++++++++++++++++ .../test/emqx_mgmt_api_configs_SUITE.erl | 4 +- 5 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 apps/emqx_management/src/proto/emqx_management_proto_v5.erl diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 6ea2b5549..6b1a8d245 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -179,7 +179,7 @@ get_sys_memory() -> end. node_info(Nodes) -> - emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)). + emqx_rpc:unwrap_erpc(emqx_management_proto_v5:node_info(Nodes)). stopped_node_info(Node) -> {Node, #{node => Node, node_status => 'stopped', role => core}}. @@ -223,7 +223,7 @@ convert_broker_info({K, V}, M) -> M#{K => iolist_to_binary(V)}. broker_info(Nodes) -> - emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)). + emqx_rpc:unwrap_erpc(emqx_management_proto_v5:broker_info(Nodes)). %%-------------------------------------------------------------------- %% Metrics and Stats @@ -330,7 +330,7 @@ kickout_client(Node, ClientId) -> kickout_clients(ClientIds) when is_list(ClientIds) -> F = fun(Node) -> - emqx_management_proto_v4:kickout_clients(Node, ClientIds) + emqx_management_proto_v5:kickout_clients(Node, ClientIds) end, Results = lists:map(F, emqx:running_nodes()), case lists:filter(fun(Res) -> Res =/= ok end, Results) of @@ -446,7 +446,7 @@ do_call_client(ClientId, Req) -> %% @private call_client(Node, ClientId, Req) -> - unwrap_rpc(emqx_management_proto_v3:call_client(Node, ClientId, Req)). + unwrap_rpc(emqx_management_proto_v5:call_client(Node, ClientId, Req)). %%-------------------------------------------------------------------- %% Subscriptions @@ -459,7 +459,7 @@ do_list_subscriptions() -> throw(not_implemented). list_subscriptions(Node) -> - unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)). + unwrap_rpc(emqx_management_proto_v5:list_subscriptions(Node)). list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ @@ -481,7 +481,7 @@ subscribe(ClientId, TopicTables) -> subscribe(emqx:running_nodes(), ClientId, TopicTables). subscribe([Node | Nodes], ClientId, TopicTables) -> - case unwrap_rpc(emqx_management_proto_v3:subscribe(Node, ClientId, TopicTables)) of + case unwrap_rpc(emqx_management_proto_v5:subscribe(Node, ClientId, TopicTables)) of {error, _} -> subscribe(Nodes, ClientId, TopicTables); {subscribe, Res} -> {subscribe, Res, Node} end; @@ -508,7 +508,7 @@ unsubscribe(ClientId, Topic) -> -spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe([Node | Nodes], ClientId, Topic) -> - case unwrap_rpc(emqx_management_proto_v3:unsubscribe(Node, ClientId, Topic)) of + case unwrap_rpc(emqx_management_proto_v5:unsubscribe(Node, ClientId, Topic)) of {error, _} -> unsubscribe(Nodes, ClientId, Topic); Re -> Re end; @@ -531,7 +531,7 @@ unsubscribe_batch(ClientId, Topics) -> -spec unsubscribe_batch([node()], emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe_batch, _} | {error, channel_not_found}. unsubscribe_batch([Node | Nodes], ClientId, Topics) -> - case unwrap_rpc(emqx_management_proto_v3:unsubscribe_batch(Node, ClientId, Topics)) of + case unwrap_rpc(emqx_management_proto_v5:unsubscribe_batch(Node, ClientId, Topics)) of {error, _} -> unsubscribe_batch(Nodes, ClientId, Topics); Re -> Re end; diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index f2e336d0f..7c5a9ffd9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -350,7 +350,7 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} end. -find_suitable_accept(Headers, Perferences) when is_list(Perferences), length(Perferences) > 0 -> +find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 -> AcceptVal = maps:get(<<"accept">>, Headers, <<"*/*">>), %% Multiple types, weighted with the quality value syntax: %% Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8 @@ -363,11 +363,11 @@ find_suitable_accept(Headers, Perferences) when is_list(Perferences), length(Per ), case lists:member(<<"*/*">>, Accepts) of true -> - {ok, lists:nth(1, Perferences)}; + {ok, lists:nth(1, Preferences)}; false -> - Found = lists:filter(fun(Accept) -> lists:member(Accept, Accepts) end, Perferences), + Found = lists:filter(fun(Accept) -> lists:member(Accept, Accepts) end, Preferences), case Found of - [] -> {error, no_suitalbe_accept}; + [] -> {error, no_suitable_accept}; _ -> {ok, lists:nth(1, Found)} end end. @@ -376,7 +376,7 @@ get_configs_v1(QueryStr) -> Node = maps:get(<<"node">>, QueryStr, node()), case lists:member(Node, emqx:running_nodes()) andalso - emqx_management_proto_v2:get_full_config(Node) + emqx_management_proto_v5:get_full_config(Node) of false -> Message = list_to_binary(io_lib:format("Bad node ~p, reason not found", [Node])), @@ -389,10 +389,13 @@ get_configs_v1(QueryStr) -> end. get_configs_v2(QueryStr) -> + Node = maps:get(<<"node">>, QueryStr, node()), Conf = case maps:find(<<"key">>, QueryStr) of - error -> emqx_conf_cli:get_config(); - {ok, Key} -> emqx_conf_cli:get_config(atom_to_binary(Key)) + error -> + emqx_management_proto_v5:get_full_config_v2(Node); + {ok, Key} -> + emqx_management_proto_v5:get_config_v2(Node, atom_to_binary(Key)) end, { 200, diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 719d0913d..3db52b08c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -515,7 +515,7 @@ list_listeners() -> lists:map(fun list_listeners/1, [Self | lists:delete(Self, emqx:running_nodes())]). list_listeners(Node) -> - wrap_rpc(emqx_management_proto_v2:list_listeners(Node)). + wrap_rpc(emqx_management_proto_v5:list_listeners(Node)). listener_status_by_id(NodeL) -> Listeners = maps:to_list(listener_status_by_id(NodeL, #{})), diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v5.erl b/apps/emqx_management/src/proto/emqx_management_proto_v5.erl new file mode 100644 index 000000000..e0a2ec6e2 --- /dev/null +++ b/apps/emqx_management/src/proto/emqx_management_proto_v5.erl @@ -0,0 +1,96 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_management_proto_v5). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + node_info/1, + broker_info/1, + list_subscriptions/1, + + list_listeners/1, + subscribe/3, + unsubscribe/3, + unsubscribe_batch/3, + + call_client/3, + + get_full_config/1, + get_full_config_v2/1, + get_config_v2/2, + + kickout_clients/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.1.1". + +-spec unsubscribe_batch(node(), emqx_types:clientid(), [emqx_types:topic()]) -> + {unsubscribe, _} | {error, _} | {badrpc, _}. +unsubscribe_batch(Node, ClientId, Topics) -> + rpc:call(Node, emqx_mgmt, do_unsubscribe_batch, [ClientId, Topics]). + +-spec node_info([node()]) -> emqx_rpc:erpc_multicall(map()). +node_info(Nodes) -> + erpc:multicall(Nodes, emqx_mgmt, node_info, [], 30000). + +-spec broker_info([node()]) -> emqx_rpc:erpc_multicall(map()). +broker_info(Nodes) -> + erpc:multicall(Nodes, emqx_mgmt, broker_info, [], 30000). + +-spec list_subscriptions(node()) -> [map()] | {badrpc, _}. +list_subscriptions(Node) -> + rpc:call(Node, emqx_mgmt, do_list_subscriptions, []). + +-spec list_listeners(node()) -> map() | {badrpc, _}. +list_listeners(Node) -> + rpc:call(Node, emqx_mgmt_api_listeners, do_list_listeners, []). + +-spec subscribe(node(), emqx_types:clientid(), emqx_types:topic_filters()) -> + {subscribe, _} | {error, atom()} | {badrpc, _}. +subscribe(Node, ClientId, TopicTables) -> + rpc:call(Node, emqx_mgmt, do_subscribe, [ClientId, TopicTables]). + +-spec unsubscribe(node(), emqx_types:clientid(), emqx_types:topic()) -> + {unsubscribe, _} | {error, _} | {badrpc, _}. +unsubscribe(Node, ClientId, Topic) -> + rpc:call(Node, emqx_mgmt, do_unsubscribe, [ClientId, Topic]). + +-spec call_client(node(), emqx_types:clientid(), term()) -> term(). +call_client(Node, ClientId, Req) -> + rpc:call(Node, emqx_mgmt, do_call_client, [ClientId, Req]). + +-spec get_full_config(node()) -> map() | list() | {badrpc, _}. +get_full_config(Node) -> + rpc:call(Node, emqx_mgmt_api_configs, get_full_config, []). + +-spec kickout_clients(node(), [emqx_types:clientid()]) -> ok | {badrpc, _}. +kickout_clients(Node, ClientIds) -> + rpc:call(Node, emqx_mgmt, do_kickout_clients, [ClientIds]). + +-spec get_full_config_v2(node()) -> map() | {badrpc, _}. +get_full_config_v2(Node) -> + rpc:call(Node, emqx_conf_cli, get_config, []). + +-spec get_config_v2(node(), binary()) -> map() | {badrpc, _}. +get_config_v2(Node, Key) -> + rpc:call(Node, emqx_conf_cli, get_config, [Key]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 43554c9ff..99048bdfd 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -273,7 +273,7 @@ t_configs_node({'init', Config}) -> Node = node(), meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), meck:expect( - emqx_management_proto_v2, + emqx_management_proto_v5, get_full_config, fun (Node0) when Node0 =:= Node -> <<"\"self\"">>; @@ -283,7 +283,7 @@ t_configs_node({'init', Config}) -> ), Config; t_configs_node({'end', _}) -> - meck:unload([emqx, emqx_management_proto_v2]); + meck:unload([emqx, emqx_management_proto_v5]); t_configs_node(_) -> Node = atom_to_list(node()), From 8a0f7bfd9974fee0bb74ca27d8efbf91861cde34 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Jul 2023 22:24:16 +0800 Subject: [PATCH 07/28] chore: update version bpapi.versions --- apps/emqx/priv/bpapi.versions | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 2d1ba49eb..21a8bec54 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -28,6 +28,7 @@ {emqx_management,2}. {emqx_management,3}. {emqx_management,4}. +{emqx_management,5}. {emqx_metrics,1}. {emqx_mgmt_api_plugins,1}. {emqx_mgmt_api_plugins,2}. From 402f75592e18495e9c73ba51990e92af4a453c1b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 17 Jul 2023 09:45:38 +0800 Subject: [PATCH 08/28] chore: move rpc call from emqx_management_proto to emqx_conf_proto --- apps/emqx/priv/bpapi.versions | 2 +- apps/emqx_conf/src/emqx_conf.erl | 18 +-- apps/emqx_conf/src/emqx_conf_app.erl | 4 +- .../src/proto/emqx_conf_proto_v3.erl | 119 ++++++++++++++++++ apps/emqx_management/src/emqx_mgmt.erl | 16 +-- .../src/emqx_mgmt_api_configs.erl | 7 +- .../src/emqx_mgmt_api_listeners.erl | 2 +- .../src/proto/emqx_management_proto_v5.erl | 96 -------------- .../test/emqx_mgmt_api_configs_SUITE.erl | 4 +- 9 files changed, 146 insertions(+), 122 deletions(-) create mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl delete mode 100644 apps/emqx_management/src/proto/emqx_management_proto_v5.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 21a8bec54..e13f60654 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -12,6 +12,7 @@ {emqx_cm,2}. {emqx_conf,1}. {emqx_conf,2}. +{emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_eviction_agent,1}. @@ -28,7 +29,6 @@ {emqx_management,2}. {emqx_management,3}. {emqx_management,4}. -{emqx_management,5}. {emqx_metrics,1}. {emqx_mgmt_api_plugins,1}. {emqx_mgmt_api_plugins,2}. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d5f39bcb0..1efeb4d69 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -71,7 +71,7 @@ get_raw(KeyPath) -> %% @doc Returns all values in the cluster. -spec get_all(emqx_utils_maps:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> - {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), + {ResL, []} = emqx_conf_proto_v3:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found @@ -79,14 +79,14 @@ get_all(KeyPath) -> get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> - emqx_conf_proto_v2:get_config(Node, KeyPath). + emqx_conf_proto_v3:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found -spec get_by_node(node(), emqx_utils_maps:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> - emqx_conf_proto_v2:get_config(Node, KeyPath, Default). + emqx_conf_proto_v3:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found -spec get_node_and_config(emqx_utils_maps:config_key_path()) -> term(). @@ -101,7 +101,7 @@ get_node_and_config(KeyPath) -> ) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update(KeyPath, UpdateReq, Opts) -> - emqx_conf_proto_v2:update(KeyPath, UpdateReq, Opts). + emqx_conf_proto_v3:update(KeyPath, UpdateReq, Opts). %% @doc Update the specified node's key path in local-override.conf. -spec update( @@ -114,7 +114,7 @@ update(KeyPath, UpdateReq, Opts) -> update(Node, KeyPath, UpdateReq, Opts0) when Node =:= node() -> emqx:update_config(KeyPath, UpdateReq, Opts0#{override_to => local}); update(Node, KeyPath, UpdateReq, Opts) -> - emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). + emqx_conf_proto_v3:update(Node, KeyPath, UpdateReq, Opts). %% @doc Mark the specified key path as tombstone tombstone(KeyPath, Opts) -> @@ -124,7 +124,7 @@ tombstone(KeyPath, Opts) -> -spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> - emqx_conf_proto_v2:remove_config(KeyPath, Opts). + emqx_conf_proto_v3:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. -spec remove(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> @@ -132,13 +132,13 @@ remove(KeyPath, Opts) -> remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); remove(Node, KeyPath, Opts) -> - emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). + emqx_conf_proto_v3:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. -spec reset(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> - emqx_conf_proto_v2:reset(KeyPath, Opts). + emqx_conf_proto_v3:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. -spec reset(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> @@ -146,7 +146,7 @@ reset(KeyPath, Opts) -> reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); reset(Node, KeyPath, Opts) -> - emqx_conf_proto_v2:reset(Node, KeyPath, Opts). + emqx_conf_proto_v3:reset(Node, KeyPath, Opts). %% @doc Called from build script. %% TODO: move to a external escript after all refactoring is done diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 0a486c829..7addb3823 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -137,7 +137,7 @@ sync_cluster_conf() -> %% @private Some core nodes are running, try to sync the cluster config from them. sync_cluster_conf2(Nodes) -> - {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes), + {Results, Failed} = emqx_conf_proto_v3:get_override_config_file(Nodes), {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results), NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0), case (Failed =/= [] orelse NotReady =/= []) of @@ -284,7 +284,7 @@ conf_sort({ok, _}, {ok, _}) -> false. sync_data_from_node(Node) -> - case emqx_conf_proto_v2:sync_data_from_node(Node) of + case emqx_conf_proto_v3:sync_data_from_node(Node) of {ok, DataBin} -> case zip:unzip(DataBin, [{cwd, emqx:data_dir()}]) of {ok, []} -> diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl new file mode 100644 index 000000000..a2719bc8e --- /dev/null +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl @@ -0,0 +1,119 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + sync_data_from_node/1, + get_config/2, + get_config/3, + get_all/1, + + update/3, + update/4, + remove_config/2, + remove_config/3, + + reset/2, + reset/3, + + get_override_config_file/1 +]). + +-export([get_hocon_config/1, get_hocon_config/2]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.1.1". + +-spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). +sync_data_from_node(Node) -> + rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. + +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath) -> + rpc:call(Node, emqx, get_config, [KeyPath]). + +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath, Default) -> + rpc:call(Node, emqx, get_config, [KeyPath, Default]). + +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). +get_all(KeyPath) -> + rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). + +-spec update( + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +update(KeyPath, UpdateReq, Opts) -> + emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). + +-spec update( + node(), + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +update(Node, KeyPath, UpdateReq, Opts) -> + rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). + +-spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +remove_config(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). + +-spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +remove_config(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). + +-spec reset(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +reset(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). + +-spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +reset(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). + +-spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). +get_override_config_file(Nodes) -> + rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). + +-spec get_hocon_config(node()) -> map() | {badrpc, _}. +get_hocon_config(Node) -> + rpc:call(Node, emqx_conf_cli, get_config, []). + +-spec get_hocon_config(node(), binary()) -> map() | {badrpc, _}. +get_hocon_config(Node, Key) -> + rpc:call(Node, emqx_conf_cli, get_config, [Key]). diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 6b1a8d245..2f261c0d5 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -179,7 +179,7 @@ get_sys_memory() -> end. node_info(Nodes) -> - emqx_rpc:unwrap_erpc(emqx_management_proto_v5:node_info(Nodes)). + emqx_rpc:unwrap_erpc(emqx_management_proto_v4:node_info(Nodes)). stopped_node_info(Node) -> {Node, #{node => Node, node_status => 'stopped', role => core}}. @@ -223,7 +223,7 @@ convert_broker_info({K, V}, M) -> M#{K => iolist_to_binary(V)}. broker_info(Nodes) -> - emqx_rpc:unwrap_erpc(emqx_management_proto_v5:broker_info(Nodes)). + emqx_rpc:unwrap_erpc(emqx_management_proto_v4:broker_info(Nodes)). %%-------------------------------------------------------------------- %% Metrics and Stats @@ -330,7 +330,7 @@ kickout_client(Node, ClientId) -> kickout_clients(ClientIds) when is_list(ClientIds) -> F = fun(Node) -> - emqx_management_proto_v5:kickout_clients(Node, ClientIds) + emqx_management_proto_v4:kickout_clients(Node, ClientIds) end, Results = lists:map(F, emqx:running_nodes()), case lists:filter(fun(Res) -> Res =/= ok end, Results) of @@ -446,7 +446,7 @@ do_call_client(ClientId, Req) -> %% @private call_client(Node, ClientId, Req) -> - unwrap_rpc(emqx_management_proto_v5:call_client(Node, ClientId, Req)). + unwrap_rpc(emqx_management_proto_v4:call_client(Node, ClientId, Req)). %%-------------------------------------------------------------------- %% Subscriptions @@ -459,7 +459,7 @@ do_list_subscriptions() -> throw(not_implemented). list_subscriptions(Node) -> - unwrap_rpc(emqx_management_proto_v5:list_subscriptions(Node)). + unwrap_rpc(emqx_management_proto_v4:list_subscriptions(Node)). list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ @@ -481,7 +481,7 @@ subscribe(ClientId, TopicTables) -> subscribe(emqx:running_nodes(), ClientId, TopicTables). subscribe([Node | Nodes], ClientId, TopicTables) -> - case unwrap_rpc(emqx_management_proto_v5:subscribe(Node, ClientId, TopicTables)) of + case unwrap_rpc(emqx_management_proto_v4:subscribe(Node, ClientId, TopicTables)) of {error, _} -> subscribe(Nodes, ClientId, TopicTables); {subscribe, Res} -> {subscribe, Res, Node} end; @@ -508,7 +508,7 @@ unsubscribe(ClientId, Topic) -> -spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe([Node | Nodes], ClientId, Topic) -> - case unwrap_rpc(emqx_management_proto_v5:unsubscribe(Node, ClientId, Topic)) of + case unwrap_rpc(emqx_management_proto_v4:unsubscribe(Node, ClientId, Topic)) of {error, _} -> unsubscribe(Nodes, ClientId, Topic); Re -> Re end; @@ -531,7 +531,7 @@ unsubscribe_batch(ClientId, Topics) -> -spec unsubscribe_batch([node()], emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe_batch, _} | {error, channel_not_found}. unsubscribe_batch([Node | Nodes], ClientId, Topics) -> - case unwrap_rpc(emqx_management_proto_v5:unsubscribe_batch(Node, ClientId, Topics)) of + case unwrap_rpc(emqx_management_proto_v4:unsubscribe_batch(Node, ClientId, Topics)) of {error, _} -> unsubscribe_batch(Nodes, ClientId, Topics); Re -> Re end; diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 7c5a9ffd9..b644a7dae 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -372,11 +372,12 @@ find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Pre end end. +%% Should deprecated json v1 since 5.2.0 get_configs_v1(QueryStr) -> Node = maps:get(<<"node">>, QueryStr, node()), case lists:member(Node, emqx:running_nodes()) andalso - emqx_management_proto_v5:get_full_config(Node) + emqx_management_proto_v4:get_full_config(Node) of false -> Message = list_to_binary(io_lib:format("Bad node ~p, reason not found", [Node])), @@ -393,9 +394,9 @@ get_configs_v2(QueryStr) -> Conf = case maps:find(<<"key">>, QueryStr) of error -> - emqx_management_proto_v5:get_full_config_v2(Node); + emqx_conf_proto_v3:get_hocon_config(Node); {ok, Key} -> - emqx_management_proto_v5:get_config_v2(Node, atom_to_binary(Key)) + emqx_conf_proto_v3:get_hocon_config(Node, atom_to_binary(Key)) end, { 200, diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index 3db52b08c..90fb1f98e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -515,7 +515,7 @@ list_listeners() -> lists:map(fun list_listeners/1, [Self | lists:delete(Self, emqx:running_nodes())]). list_listeners(Node) -> - wrap_rpc(emqx_management_proto_v5:list_listeners(Node)). + wrap_rpc(emqx_management_proto_v4:list_listeners(Node)). listener_status_by_id(NodeL) -> Listeners = maps:to_list(listener_status_by_id(NodeL, #{})), diff --git a/apps/emqx_management/src/proto/emqx_management_proto_v5.erl b/apps/emqx_management/src/proto/emqx_management_proto_v5.erl deleted file mode 100644 index e0a2ec6e2..000000000 --- a/apps/emqx_management/src/proto/emqx_management_proto_v5.erl +++ /dev/null @@ -1,96 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_management_proto_v5). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - - node_info/1, - broker_info/1, - list_subscriptions/1, - - list_listeners/1, - subscribe/3, - unsubscribe/3, - unsubscribe_batch/3, - - call_client/3, - - get_full_config/1, - get_full_config_v2/1, - get_config_v2/2, - - kickout_clients/2 -]). - --include_lib("emqx/include/bpapi.hrl"). - -introduced_in() -> - "5.1.1". - --spec unsubscribe_batch(node(), emqx_types:clientid(), [emqx_types:topic()]) -> - {unsubscribe, _} | {error, _} | {badrpc, _}. -unsubscribe_batch(Node, ClientId, Topics) -> - rpc:call(Node, emqx_mgmt, do_unsubscribe_batch, [ClientId, Topics]). - --spec node_info([node()]) -> emqx_rpc:erpc_multicall(map()). -node_info(Nodes) -> - erpc:multicall(Nodes, emqx_mgmt, node_info, [], 30000). - --spec broker_info([node()]) -> emqx_rpc:erpc_multicall(map()). -broker_info(Nodes) -> - erpc:multicall(Nodes, emqx_mgmt, broker_info, [], 30000). - --spec list_subscriptions(node()) -> [map()] | {badrpc, _}. -list_subscriptions(Node) -> - rpc:call(Node, emqx_mgmt, do_list_subscriptions, []). - --spec list_listeners(node()) -> map() | {badrpc, _}. -list_listeners(Node) -> - rpc:call(Node, emqx_mgmt_api_listeners, do_list_listeners, []). - --spec subscribe(node(), emqx_types:clientid(), emqx_types:topic_filters()) -> - {subscribe, _} | {error, atom()} | {badrpc, _}. -subscribe(Node, ClientId, TopicTables) -> - rpc:call(Node, emqx_mgmt, do_subscribe, [ClientId, TopicTables]). - --spec unsubscribe(node(), emqx_types:clientid(), emqx_types:topic()) -> - {unsubscribe, _} | {error, _} | {badrpc, _}. -unsubscribe(Node, ClientId, Topic) -> - rpc:call(Node, emqx_mgmt, do_unsubscribe, [ClientId, Topic]). - --spec call_client(node(), emqx_types:clientid(), term()) -> term(). -call_client(Node, ClientId, Req) -> - rpc:call(Node, emqx_mgmt, do_call_client, [ClientId, Req]). - --spec get_full_config(node()) -> map() | list() | {badrpc, _}. -get_full_config(Node) -> - rpc:call(Node, emqx_mgmt_api_configs, get_full_config, []). - --spec kickout_clients(node(), [emqx_types:clientid()]) -> ok | {badrpc, _}. -kickout_clients(Node, ClientIds) -> - rpc:call(Node, emqx_mgmt, do_kickout_clients, [ClientIds]). - --spec get_full_config_v2(node()) -> map() | {badrpc, _}. -get_full_config_v2(Node) -> - rpc:call(Node, emqx_conf_cli, get_config, []). - --spec get_config_v2(node(), binary()) -> map() | {badrpc, _}. -get_config_v2(Node, Key) -> - rpc:call(Node, emqx_conf_cli, get_config, [Key]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 99048bdfd..f853adebf 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -273,7 +273,7 @@ t_configs_node({'init', Config}) -> Node = node(), meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), meck:expect( - emqx_management_proto_v5, + emqx_management_proto_v4, get_full_config, fun (Node0) when Node0 =:= Node -> <<"\"self\"">>; @@ -283,7 +283,7 @@ t_configs_node({'init', Config}) -> ), Config; t_configs_node({'end', _}) -> - meck:unload([emqx, emqx_management_proto_v5]); + meck:unload([emqx, emqx_management_proto_v4]); t_configs_node(_) -> Node = atom_to_list(node()), From 74532a64e61845de54ac847c2a9c632a6f20f798 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Jul 2023 20:50:23 +0800 Subject: [PATCH 09/28] chore: update apps/emqx_management/src/emqx_mgmt_api_configs.erl --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index b644a7dae..4d6059d7d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -372,7 +372,13 @@ find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Pre end end. -%% Should deprecated json v1 since 5.2.0 +%% To return a JSON formatted configuration file, which is used to be compatible with the already +%% implemented `GET /configs` in the old versions 5.0 and 5.1. +%% +%% In e5.1.1, we support to return a hocon configuration file by `get_configs_v2/1`. It's more +%% useful for the user to read or reload the configuration file via HTTP API. +%% +%% The `get_configs_v1/1` should be deprecated since 5.2.0. get_configs_v1(QueryStr) -> Node = maps:get(<<"node">>, QueryStr, node()), case From 439eb4d029777f1f414b385eca1ce5430ffb3b88 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Jul 2023 08:51:26 +0800 Subject: [PATCH 10/28] test: cover the node params for `get /configs` endpoint --- .../test/emqx_mgmt_api_configs_SUITE.erl | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index f853adebf..887d88188 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -272,18 +272,22 @@ t_dashboard(_Config) -> t_configs_node({'init', Config}) -> Node = node(), meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), - meck:expect( - emqx_management_proto_v4, - get_full_config, - fun - (Node0) when Node0 =:= Node -> <<"\"self\"">>; - (other_node) -> <<"\"other\"">>; - (bad_node) -> {badrpc, bad} - end - ), + F = fun + (Node0) when Node0 =:= Node -> <<"\"self\"">>; + (other_node) -> <<"\"other\"">>; + (bad_node) -> {badrpc, bad} + end, + F2 = fun + (Node0, _) when Node0 =:= Node -> <<"log=1">>; + (other_node, _) -> <<"log=2">>; + (bad_node, _) -> {badrpc, bad} + end, + meck:expect(emqx_management_proto_v4, get_full_config, F), + meck:expect(emqx_conf_proto_v3, get_hocon_config, F2), + meck:expect(hocon_pp, do, fun(Conf, _) -> Conf end), Config; t_configs_node({'end', _}) -> - meck:unload([emqx, emqx_management_proto_v4]); + meck:unload([emqx, emqx_management_proto_v4, emqx_conf_proto_v3, hocon_pp]); t_configs_node(_) -> Node = atom_to_list(node()), @@ -296,7 +300,10 @@ t_configs_node(_) -> {_, _, Body} = ExpRes, ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), - ?assertMatch({error, {_, 500, _}}, get_configs_with_json("bad_node")). + ?assertMatch({error, {_, 500, _}}, get_configs_with_json("bad_node")), + + ?assertEqual({ok, #{<<"log">> => 1}}, get_configs_with_binary("log", Node)), + ?assertEqual({ok, #{<<"log">> => 2}}, get_configs_with_binary("log", "other_node")). %% v2 version binary t_configs_key(_Config) -> @@ -386,12 +393,16 @@ get_configs_with_json(Node, Opts) -> end. get_configs_with_binary(Key) -> + get_configs_with_binary(Key, node()). + +get_configs_with_binary(Key, Node) -> + Path0 = "configs?node=" ++ Node, Path = case Key of - undefined -> ["configs"]; - _ -> ["configs?key=" ++ Key] + undefined -> Path0; + _ -> Path0 ++ "&key=" ++ Key end, - URI = emqx_mgmt_api_test_util:api_path(Path), + URI = emqx_mgmt_api_test_util:api_path([Path]), Auth = emqx_mgmt_api_test_util:auth_header_(), Headers = [{"accept", "text/plain"}, Auth], case emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) of From 0524c3b3b17719677035ef1768fe08b82b8ca5fd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 19 Jul 2023 11:05:03 +0800 Subject: [PATCH 11/28] test: configs SUITE crash --- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 887d88188..8ff3c1794 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -393,7 +393,7 @@ get_configs_with_json(Node, Opts) -> end. get_configs_with_binary(Key) -> - get_configs_with_binary(Key, node()). + get_configs_with_binary(Key, atom_to_list(node())). get_configs_with_binary(Key, Node) -> Path0 = "configs?node=" ++ Node, From 7ac8715358226ee9359f7a4bc52ab05314848714 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Jul 2023 14:43:50 +0800 Subject: [PATCH 12/28] fix(hstream): handle the `unavailable` error code The "unavailable" error code just means that the HStream is warming up or busy, HStream guarantees that a retry will work well. --- .../src/emqx_bridge_hstreamdb_connector.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 1e2ebbfb1..1763b252a 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -309,6 +309,13 @@ do_append_records(false, Producer, Record) -> msg => "HStreamDB producer sync append success", record => Record }); + %% the HStream is warming up or buzy, something are not ready yet, retry after a while + {error, {unavailable, _} = Reason} -> + {error, + {recoverable_error, #{ + msg => "HStreamDB is warming up or buzy, will retry after a moment", + reason => Reason + }}}; {error, Reason} = Err -> ?tp( hstreamdb_connector_query_return, From e76ba760c9691b4188da5e5e064f02591d6ef6cf Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 19 Jul 2023 11:47:06 -0300 Subject: [PATCH 13/28] fix(oracle): discard nested tokens when checking table Fixes https://emqx.atlassian.net/browse/EMQX-10597 --- .../test/emqx_bridge_oracle_SUITE.erl | 20 +++++++++++++++++++ apps/emqx_oracle/src/emqx_oracle.app.src | 2 +- apps/emqx_oracle/src/emqx_oracle.erl | 20 +++++++++++++++++-- changes/ee/fix-11307.en.md | 1 + 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 changes/ee/fix-11307.en.md diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index 5c6eddb39..43eb22903 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -162,6 +162,9 @@ delete_all_bridges() -> sql_insert_template_for_bridge() -> "INSERT INTO mqtt_test(topic, msgid, payload, retain) VALUES (${topic}, ${id}, ${payload}, ${retain})". +sql_insert_template_with_nested_token_for_bridge() -> + "INSERT INTO mqtt_test(topic, msgid, payload, retain) VALUES (${topic}, ${id}, ${payload.msg}, ${retain})". + sql_create_table() -> "CREATE TABLE mqtt_test (topic VARCHAR2(255), msgid VARCHAR2(64), payload NCLOB, retain NUMBER(1))". @@ -533,6 +536,23 @@ t_start_stop(Config) -> ), ok. +t_probe_with_nested_tokens(Config) -> + ResourceId = resource_id(Config), + reset_table(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config, #{ + <<"sql">> => sql_insert_template_with_nested_token_for_bridge() + }) + ), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ). + t_on_get_status(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index a5ca822e8..be3ed3276 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 5a7f8d752..b33122fb7 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -9,6 +9,10 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-define(UNHEALTHY_TARGET_MSG, + "Oracle table is invalid. Please check if the table exists in Oracle Database." +). + %%==================================================================== %% Exports %%==================================================================== @@ -239,7 +243,7 @@ on_get_status(_InstId, #{pool_name := Pool} = State) -> {connected, NState}; {error, {undefined_table, NState}} -> %% return new state indicating that we are connected but the target table is not created - {disconnected, NState, unhealthy_target}; + {disconnected, NState, {unhealthy_target, ?UNHEALTHY_TARGET_MSG}}; {error, _Reason} -> %% do not log error, it is logged in prepare_sql_to_conn connecting @@ -408,7 +412,19 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], TokensMap, Statements) whe Error end. -check_if_table_exists(Conn, SQL, Tokens) -> +check_if_table_exists(Conn, SQL, Tokens0) -> + % Discard nested tokens for checking if table exist. As payload here is defined as + % a single string, it would fail if Token is, for instance, ${payload.msg}, causing + % bridge probe to fail. + Tokens = lists:map( + fun + ({var, [Token | _DiscardedDeepTokens]}) -> + {var, [Token]}; + (Token) -> + Token + end, + Tokens0 + ), {Event, _Headers} = emqx_rule_events:eventmsg_publish( emqx_message:make(<<"t/opic">>, "test query") ), diff --git a/changes/ee/fix-11307.en.md b/changes/ee/fix-11307.en.md new file mode 100644 index 000000000..9b164f17d --- /dev/null +++ b/changes/ee/fix-11307.en.md @@ -0,0 +1 @@ +Fixed check for table existence to return a more friendly message in the Oracle bridge. From b9b11d8f4ddf04f456a3e47eacac3ba00ddfcc00 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Jul 2023 14:59:25 -0300 Subject: [PATCH 14/28] fix(machine_boot): use shared list of reboot apps and add bridges to reboot list --- Makefile | 2 +- ...emqx_bridge_pulsar_impl_producer_SUITE.erl | 6 + apps/emqx_machine/priv/reboot_lists.eterm | 110 ++++++++++++ apps/emqx_machine/src/emqx_machine_boot.erl | 94 ++++++---- .../test/emqx_schema_registry_SUITE.erl | 17 +- mix.exs | 170 +++++------------- rebar.config.erl | 143 ++++----------- 7 files changed, 267 insertions(+), 275 deletions(-) create mode 100644 apps/emqx_machine/priv/reboot_lists.eterm diff --git a/Makefile b/Makefile index c51966232..948d1d20b 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,7 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt)))) .PHONY: fmt fmt: $(REBAR) - @$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}' + @$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,priv,test}/**/*.{erl,hrl,app.src,eterm}' @$(SCRIPTS)/erlfmt -w 'rebar.config.erl' @mix format diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index 15d4b63d4..4f0f73732 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -547,6 +547,7 @@ start_cluster(Cluster) -> emqx_common_test_helpers:start_slave(Name, Opts) || {Name, Opts} <- Cluster ], + NumNodes = length(Nodes), on_exit(fun() -> emqx_utils:pmap( fun(N) -> @@ -556,6 +557,11 @@ start_cluster(Cluster) -> Nodes ) end), + {ok, _} = snabbkaffe:block_until( + %% -1 because only those that join the first node will emit the event. + ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}), + 30_000 + ), Nodes. kill_resource_managers() -> diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm new file mode 100644 index 000000000..62298697c --- /dev/null +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -0,0 +1,110 @@ +%% -*- mode: erlang; -*- +#{ + %% must always be of type `load' + db_apps => + [ + mnesia_rocksdb, + mnesia, + mria, + ekka + ], + system_apps => + [ + kernel, + sasl, + crypto, + public_key, + asn1, + syntax_tools, + ssl, + os_mon, + inets, + compiler, + runtime_tools, + redbug, + xmerl, + {hocon, load}, + telemetry + ], + %% must always be of type `load' + common_business_apps => + [ + emqx, + emqx_conf, + + esasl, + observer_cli, + tools, + covertool, + %% started by emqx_machine + system_monitor, + emqx_utils, + emqx_http_lib, + emqx_resource, + emqx_connector, + emqx_authn, + emqx_authz, + emqx_auto_subscribe, + emqx_gateway, + emqx_gateway_stomp, + emqx_gateway_mqttsn, + emqx_gateway_coap, + emqx_gateway_lwm2m, + emqx_gateway_exproto, + emqx_exhook, + emqx_bridge, + emqx_bridge_mqtt, + emqx_bridge_http, + emqx_rule_engine, + emqx_modules, + emqx_management, + emqx_dashboard, + emqx_retainer, + emqx_prometheus, + emqx_psk, + emqx_slow_subs, + emqx_mongodb, + emqx_redis, + emqx_mysql, + emqx_plugins, + quicer, + bcrypt, + jq, + observer + ], + %% must always be of type `load' + ee_business_apps => + [ + emqx_license, + emqx_enterprise, + emqx_bridge_kafka, + emqx_bridge_pulsar, + emqx_bridge_gcp_pubsub, + emqx_bridge_cassandra, + emqx_bridge_opents, + emqx_bridge_clickhouse, + emqx_bridge_dynamo, + emqx_bridge_hstreamdb, + emqx_bridge_influxdb, + emqx_bridge_iotdb, + emqx_bridge_matrix, + emqx_bridge_mongodb, + emqx_bridge_mysql, + emqx_bridge_pgsql, + emqx_bridge_redis, + emqx_bridge_rocketmq, + emqx_bridge_tdengine, + emqx_bridge_timescale, + emqx_bridge_sqlserver, + emqx_oracle, + emqx_bridge_oracle, + emqx_bridge_rabbitmq, + emqx_schema_registry, + emqx_eviction_agent, + emqx_node_rebalance, + emqx_ft + ], + %% must always be of type `load' + ce_business_apps => + [emqx_telemetry] +}. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 361566e96..f2490ddef 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -16,6 +16,7 @@ -module(emqx_machine_boot). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([post_boot/0]). -export([stop_apps/0, ensure_apps_started/0]). @@ -24,7 +25,6 @@ -export([stop_port_apps/0]). -dialyzer({no_match, [basic_reboot_apps/0]}). --dialyzer({no_match, [basic_reboot_apps_edition/1]}). -ifdef(TEST). -export([sorted_reboot_apps/1, reboot_apps/0]). @@ -94,7 +94,8 @@ stop_one_app(App) -> ensure_apps_started() -> ?SLOG(notice, #{msg => "(re)starting_emqx_apps"}), - lists:foreach(fun start_one_app/1, sorted_reboot_apps()). + lists:foreach(fun start_one_app/1, sorted_reboot_apps()), + ?tp(emqx_machine_boot_apps_started, #{}). start_one_app(App) -> ?SLOG(debug, #{msg => "starting_app", app => App}), @@ -128,41 +129,62 @@ reboot_apps() -> BaseRebootApps ++ ConfigApps. basic_reboot_apps() -> - ?BASIC_REBOOT_APPS ++ - [ - emqx_prometheus, - emqx_modules, - emqx_dashboard, - emqx_connector, - emqx_gateway, - emqx_resource, - emqx_rule_engine, - emqx_bridge, - emqx_management, - emqx_retainer, - emqx_exhook, - emqx_authn, - emqx_authz, - emqx_slow_subs, - emqx_auto_subscribe, - emqx_plugins, - emqx_psk - ] ++ basic_reboot_apps_edition(emqx_release:edition()). + PrivDir = code:priv_dir(emqx_machine), + RebootListPath = filename:join([PrivDir, "reboot_lists.eterm"]), + {ok, [ + #{ + common_business_apps := CommonBusinessApps0, + ee_business_apps := EEBusinessApps, + ce_business_apps := CEBusinessApps + } + ]} = file:consult(RebootListPath), + Filters0 = maps:from_list([ + {App, is_app(App)} + || App <- [quicer, bcrypt, jq, observer] + ]), + CommonBusinessApps = + filter( + CommonBusinessApps0, + %% We don't need to restart these + Filters0#{ + system_monitor => false, + observer => false, + quicer => false + } + ), + EditionSpecificApps = + case emqx_release:edition() of + ee -> EEBusinessApps; + ce -> CEBusinessApps; + _ -> [] + end, + BusinessApps = CommonBusinessApps ++ EditionSpecificApps, + ?BASIC_REBOOT_APPS ++ BusinessApps. -basic_reboot_apps_edition(ce) -> - [emqx_telemetry]; -basic_reboot_apps_edition(ee) -> - [ - emqx_license, - emqx_s3, - emqx_ft, - emqx_eviction_agent, - emqx_node_rebalance, - emqx_schema_registry - ]; -%% unexcepted edition, should not happen -basic_reboot_apps_edition(_) -> - []. +filter(AppList, Filters) -> + lists:foldr( + fun(App, Acc) -> + AppName = + case App of + {Name, _Type} -> Name; + Name when is_atom(Name) -> Name + end, + ShouldKeep = maps:get(AppName, Filters, true), + case ShouldKeep of + true -> [App | Acc]; + false -> Acc + end + end, + [], + AppList + ). + +is_app(Name) -> + case application:load(Name) of + ok -> true; + {error, {already_loaded, _}} -> true; + _ -> false + end. sorted_reboot_apps() -> Apps = [{App, app_deps(App)} || App <- reboot_apps()], diff --git a/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl b/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl index 322d3ab75..0db2ea3c6 100644 --- a/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl +++ b/apps/emqx_schema_registry/test/emqx_schema_registry_SUITE.erl @@ -364,11 +364,7 @@ cluster(Config) -> {load_schema, true}, {start_autocluster, true}, {schema_mod, emqx_enterprise_schema}, - %% need to restart schema registry app in the tests so - %% that it re-registers the config handler that is lost - %% when emqx_conf restarts during join. - {env, [{emqx_machine, applications, [emqx_schema_registry]}]}, - {load_apps, [emqx_machine | ?APPS]}, + {load_apps, [emqx_machine]}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), @@ -388,6 +384,7 @@ start_cluster(Cluster) -> emqx_common_test_helpers:start_slave(Name, Opts) || {Name, Opts} <- Cluster ], + NumNodes = length(Nodes), on_exit(fun() -> emqx_utils:pmap( fun(N) -> @@ -397,7 +394,11 @@ start_cluster(Cluster) -> Nodes ) end), - erpc:multicall(Nodes, mria_rlog, wait_for_shards, [[?SCHEMA_REGISTRY_SHARD], 30_000]), + {ok, _} = snabbkaffe:block_until( + %% -1 because only those that join the first node will emit the event. + ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}), + 30_000 + ), Nodes. wait_for_cluster_rpc(Node) -> @@ -658,7 +659,7 @@ t_cluster_serde_build(Config) -> Nodes = [N1, N2 | _] = start_cluster(Cluster), NumNodes = length(Nodes), wait_for_cluster_rpc(N2), - ?assertEqual( + ?assertMatch( ok, erpc:call(N2, emqx_schema_registry, add_schema, [SerdeName, Schema]) ), @@ -687,7 +688,7 @@ t_cluster_serde_build(Config) -> {ok, SRef1} = snabbkaffe:subscribe( ?match_event(#{?snk_kind := schema_registry_serdes_deleted}), NumNodes, - 5_000 + 10_000 ), ?assertEqual( ok, diff --git a/mix.exs b/mix.exs index fa2b291d8..5094ed07c 100644 --- a/mix.exs +++ b/mix.exs @@ -297,6 +297,7 @@ defmodule EMQXUmbrella.MixProject do [ applications: applications(edition_type), skip_mode_validation_for: [ + :emqx_mix, :emqx_gateway, :emqx_gateway_stomp, :emqx_gateway_mqttsn, @@ -316,7 +317,10 @@ defmodule EMQXUmbrella.MixProject do :emqx_auto_subscribe, :emqx_slow_subs, :emqx_plugins, - :emqx_ft + :emqx_ft, + :emqx_s3, + :emqx_durable_storage, + :rabbit_common ], steps: steps, strip_beams: false @@ -326,137 +330,57 @@ defmodule EMQXUmbrella.MixProject do end def applications(edition_type) do - system_apps = [ - crypto: :permanent, - public_key: :permanent, - asn1: :permanent, - syntax_tools: :permanent, - ssl: :permanent, - os_mon: :permanent, - inets: :permanent, - compiler: :permanent, - runtime_tools: :permanent, - redbug: :permanent, - xmerl: :permanent, - hocon: :load, - telemetry: :permanent - ] + {:ok, + [ + %{ + db_apps: db_apps, + system_apps: system_apps, + common_business_apps: common_business_apps, + ee_business_apps: ee_business_apps, + ce_business_apps: ce_business_apps + } + ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm") - db_apps = - if enable_rocksdb?() do - [:mnesia_rocksdb] + db_apps = filter(db_apps, %{mnesia_rocksdb: enable_rocksdb?()}) + + common_business_apps = + filter(common_business_apps, %{ + quicer: enable_quicer?(), + bcrypt: enable_bcrypt?(), + jq: enable_jq?(), + observer: is_app?(:observer) + }) + + edition_specific_apps = + if edition_type == :enterprise do + ee_business_apps else - [] - end ++ - [ - :mnesia, - :mria, - :ekka - ] + ce_business_apps + end - business_apps = - [ - :emqx, - :emqx_conf, - :esasl, - :observer_cli, - :tools, - :covertool, - :system_monitor, - :emqx_utils, - :emqx_http_lib, - :emqx_resource, - :emqx_connector, - :emqx_authn, - :emqx_authz, - :emqx_auto_subscribe, - :emqx_gateway, - :emqx_gateway_stomp, - :emqx_gateway_mqttsn, - :emqx_gateway_coap, - :emqx_gateway_lwm2m, - :emqx_gateway_exproto, - :emqx_exhook, - :emqx_bridge, - :emqx_bridge_mqtt, - :emqx_bridge_http, - :emqx_rule_engine, - :emqx_modules, - :emqx_management, - :emqx_dashboard, - :emqx_retainer, - :emqx_prometheus, - :emqx_psk, - :emqx_slow_subs, - :emqx_mongodb, - :emqx_redis, - :emqx_mysql, - :emqx_plugins, - :emqx_mix - ] ++ - if enable_quicer?() do - [:quicer] - else - [] - end ++ - if enable_bcrypt?() do - [:bcrypt] - else - [] - end ++ - if enable_jq?() do - [:jq] - else - [] - end ++ - if(is_app(:observer), - do: [:observer], - else: [] - ) ++ - case edition_type do - :enterprise -> - [ - :emqx_license, - :emqx_enterprise, - :emqx_bridge_kafka, - :emqx_bridge_pulsar, - :emqx_bridge_gcp_pubsub, - :emqx_bridge_cassandra, - :emqx_bridge_opents, - :emqx_bridge_clickhouse, - :emqx_bridge_dynamo, - :emqx_bridge_hstreamdb, - :emqx_bridge_influxdb, - :emqx_bridge_iotdb, - :emqx_bridge_matrix, - :emqx_bridge_mongodb, - :emqx_bridge_mysql, - :emqx_bridge_pgsql, - :emqx_bridge_redis, - :emqx_bridge_rocketmq, - :emqx_bridge_tdengine, - :emqx_bridge_timescale, - :emqx_bridge_sqlserver, - :emqx_oracle, - :emqx_bridge_oracle, - :emqx_bridge_rabbitmq, - :emqx_schema_registry, - :emqx_eviction_agent, - :emqx_node_rebalance, - :emqx_ft - ] + business_apps = common_business_apps ++ edition_specific_apps - _ -> - [:emqx_telemetry] - end - - system_apps ++ + Enum.map(system_apps, fn app -> + if is_atom(app), do: {app, :permanent}, else: app + end) ++ Enum.map(db_apps, &{&1, :load}) ++ [emqx_machine: :permanent] ++ Enum.map(business_apps, &{&1, :load}) end - defp is_app(name) do + defp filter(apps, filters) do + Enum.filter(apps, fn app -> + app_name = + case app do + {app_name, _type} -> app_name + app_name when is_atom(app_name) -> app_name + end + + Map.get(filters, app_name, true) + end) + end + + defp is_app?(name) do case Application.load(name) do :ok -> true diff --git a/rebar.config.erl b/rebar.config.erl index f33d84006..a326d6fdc 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -386,85 +386,48 @@ overlay_vars_pkg(pkg) -> ]. relx_apps(ReleaseType, Edition) -> - SystemApps = - [ - kernel, - sasl, - crypto, - public_key, - asn1, - syntax_tools, - ssl, - os_mon, - inets, - compiler, - runtime_tools, - redbug, - xmerl, - {hocon, load}, - telemetry - ], - DBApps = - [mnesia_rocksdb || is_rocksdb_supported()] ++ - [ - mnesia, - mria, - ekka - ], - BusinessApps = - [ - emqx, - emqx_conf, - - esasl, - observer_cli, - tools, - covertool, - % started by emqx_machine - system_monitor, - emqx_utils, - emqx_http_lib, - emqx_resource, - emqx_connector, - emqx_authn, - emqx_authz, - emqx_auto_subscribe, - emqx_gateway, - emqx_gateway_stomp, - emqx_gateway_mqttsn, - emqx_gateway_coap, - emqx_gateway_lwm2m, - emqx_gateway_exproto, - emqx_exhook, - emqx_bridge, - emqx_bridge_mqtt, - emqx_bridge_http, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard, - emqx_retainer, - emqx_prometheus, - emqx_psk, - emqx_slow_subs, - emqx_mongodb, - emqx_redis, - emqx_mysql, - emqx_plugins - ] ++ - [quicer || is_quicer_supported()] ++ - [bcrypt || provide_bcrypt_release(ReleaseType)] ++ - %% Started automatically when needed (only needs to be started when the - %% port implementation is used) - [jq || is_jq_supported()] ++ - [observer || is_app(observer)] ++ - relx_apps_per_edition(Edition), + {ok, [ + #{ + db_apps := DBApps0, + system_apps := SystemApps, + common_business_apps := CommonBusinessApps0, + ee_business_apps := EEBusinessApps, + ce_business_apps := CEBusinessApps + } + ]} = file:consult("apps/emqx_machine/priv/reboot_lists.eterm"), + DBApps = filter(DBApps0, #{mnesia_rocksdb => is_rocksdb_supported()}), + CommonBusinessApps = + filter(CommonBusinessApps0, #{ + quicer => is_quicer_supported(), + bcrypt => provide_bcrypt_release(ReleaseType), + jq => is_jq_supported(), + observer => is_app(observer) + }), + EditionSpecificApps = + case Edition of + ee -> EEBusinessApps; + ce -> CEBusinessApps + end, + BusinessApps = CommonBusinessApps ++ EditionSpecificApps, SystemApps ++ %% EMQX starts the DB and the business applications: [{App, load} || App <- DBApps] ++ [emqx_machine] ++ [{App, load} || App <- BusinessApps]. +filter(AppList, Filters) -> + lists:filter( + fun(App) -> + AppName = + case App of + {Name, _Type} -> Name; + Name when is_atom(Name) -> Name + end, + maps:get(AppName, Filters, true) + end, + AppList + ). + is_app(Name) -> case application:load(Name) of ok -> true; @@ -472,40 +435,6 @@ is_app(Name) -> _ -> false end. -relx_apps_per_edition(ee) -> - [ - emqx_license, - emqx_enterprise, - emqx_bridge_kafka, - emqx_bridge_pulsar, - emqx_bridge_gcp_pubsub, - emqx_bridge_cassandra, - emqx_bridge_opents, - emqx_bridge_clickhouse, - emqx_bridge_dynamo, - emqx_bridge_hstreamdb, - emqx_bridge_influxdb, - emqx_bridge_iotdb, - emqx_bridge_matrix, - emqx_bridge_mongodb, - emqx_bridge_mysql, - emqx_bridge_pgsql, - emqx_bridge_redis, - emqx_bridge_rocketmq, - emqx_bridge_tdengine, - emqx_bridge_timescale, - emqx_bridge_sqlserver, - emqx_oracle, - emqx_bridge_oracle, - emqx_bridge_rabbitmq, - emqx_schema_registry, - emqx_eviction_agent, - emqx_node_rebalance, - emqx_ft - ]; -relx_apps_per_edition(ce) -> - [emqx_telemetry]. - relx_overlay(ReleaseType, Edition) -> [ {mkdir, "log/"}, From 3cd223ff5bd30d9a4cbde93de95aab4b408ce16f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 20 Jul 2023 14:52:49 +0800 Subject: [PATCH 15/28] chore: add more detail msg for merging failed --- apps/emqx_conf/src/emqx_conf_cli.erl | 83 ++++++++++++------- .../src/emqx_mgmt_api_configs.erl | 3 +- .../src/emqx_node_rebalance_api.erl | 2 +- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index b0a1a414d..f0106fed1 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -213,10 +213,20 @@ load_config(Bin, ReplaceOrMerge) when is_binary(Bin) -> load_config_from_raw(RawConf, ReplaceOrMerge) -> case check_config(RawConf) of ok -> - lists:foreach( - fun({K, V}) -> update_config_cluster(K, V, ReplaceOrMerge) end, - to_sorted_list(RawConf) - ); + Error = + lists:filtermap( + fun({K, V}) -> + case update_config_cluster(K, V, ReplaceOrMerge) of + ok -> false; + {error, Msg} -> {true, Msg} + end + end, + to_sorted_list(RawConf) + ), + case iolist_to_binary(Error) of + <<"">> -> ok; + ErrorBin -> {error, ErrorBin} + end; {error, ?UPDATE_READONLY_KEYS_PROHIBITED = Reason} -> emqx_ctl:warning("load config failed~n~ts~n", [Reason]), emqx_ctl:warning( @@ -234,34 +244,51 @@ load_config_from_raw(RawConf, ReplaceOrMerge) -> {error, Errors} end. -update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) -> - check_res(Key, emqx_authz:merge(Conf)); -update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) -> - check_res(Key, emqx_authn:merge_config(Conf)); -update_config_cluster(Key, NewConf, merge) -> +update_config_cluster(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> + check_res(Key, emqx_authz:merge(Conf), Conf, Mode); +update_config_cluster(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> + check_res(Key, emqx_authn:merge_config(Conf), Conf, Mode); +update_config_cluster(Key, NewConf, merge = Mode) -> Merged = merge_conf(Key, NewConf), - check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS)); -update_config_cluster(Key, Value, replace) -> - check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS)). + check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Mode); +update_config_cluster(Key, Value, replace = Mode) -> + check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Mode). -define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}). -update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) -> - check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS)); -update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge) -> - check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS)); -update_config_local(Key, NewConf, merge) -> +update_config_local(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> + check_res(node(), Key, emqx_authz:merge_local(Conf, ?LOCAL_OPTIONS), Conf, Mode); +update_config_local(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY = Key, Conf, merge = Mode) -> + check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Mode); +update_config_local(Key, NewConf, merge = Mode) -> Merged = merge_conf(Key, NewConf), - check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS)); -update_config_local(Key, Value, replace) -> - check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS)). + check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Mode); +update_config_local(Key, Value, replace = Mode) -> + check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Mode). -check_res(Key, Res) -> check_res(cluster, Key, Res). -check_res(Mode, Key, {ok, _} = Res) -> - emqx_ctl:print("load ~ts in ~p ok~n", [Key, Mode]), - Res; -check_res(_Mode, Key, {error, Reason} = Res) -> - emqx_ctl:warning("load ~ts failed~n~p~n", [Key, Reason]), - Res. +check_res(Key, Res, Conf, Mode) -> check_res(cluster, Key, Res, Conf, Mode). +check_res(Node, Key, {ok, _}, _Conf, _Mode) -> + emqx_ctl:print("load ~ts on ~p ok~n", [Key, Node]), + ok; +check_res(_Node, Key, {error, Reason}, Conf, Mode) -> + Warning = io_lib:format("~ts ~ts failed~n~p~n", [Mode, Key, Reason]), + emqx_ctl:warning(Warning, []), + ActiveMsg = "--------active configuration--------\n", + Active = hocon_pp:do(#{Key => emqx_conf:get_raw([Key])}, #{}), + FailedMsg = io_lib:format("--------failed to ~ts with----------~n", [Mode]), + New = hocon_pp:do(#{Key => Conf}, #{}), + SuggestMsg = suggest_msg(Mode), + Msg = iolist_to_binary([ActiveMsg, Active, FailedMsg, New, SuggestMsg]), + emqx_ctl:print("~ts", [Msg]), + {error, iolist_to_binary([Warning, Msg])}. + +suggest_msg(merge) -> + "Merge conflict with the active Key.\n" + "Suggest to use the replace option to completely replace the configuration,\n" + "instead of merging conflict.\n\n"; +suggest_msg(replace) -> + "Replace failed with an incomplete configuration.\n" + "Suggest to use the merge option to only update sub configuration,\n" + "instead of replacing with an incomplete configuration.\n\n". check_config(Conf) -> case check_keys_is_not_readonly(Conf) of @@ -349,7 +376,7 @@ filter_readonly_config(Raw) -> reload_config(AllConf, ReplaceOrMerge) -> Fold = fun({Key, Conf}, Acc) -> case update_config_local(Key, Conf, ReplaceOrMerge) of - {ok, _} -> + ok -> Acc; Error -> ?SLOG(error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 4d6059d7d..5edf8c564 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -346,8 +346,7 @@ configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> case emqx_conf_cli:load_config(Conf, Mode) of ok -> {200}; - {error, [{_, Reason}]} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}}; - {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} + {error, Msg} -> {400, #{<<"content-type">> => <<"text/plain">>}, Msg} end. find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 -> diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index fb27c0a30..430ad1e34 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -506,7 +506,7 @@ fields(local_status_enabled) -> )}, {"process", mk( - hoconsc:union([rebalance, evacuation]), + hoconsc:enum([rebalance, evacuation]), #{ desc => ?DESC(local_status_process), required => true From 89ea40a8b7e1b624306b99b8ec16197f93b0021b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Jul 2023 09:13:48 -0300 Subject: [PATCH 16/28] refactor: apply review remarks --- apps/emqx_machine/src/emqx_machine_boot.erl | 39 +++------------- mix.exs | 49 ++++++++++----------- rebar.config.erl | 38 ++++++---------- 3 files changed, 43 insertions(+), 83 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index f2490ddef..cdb2c24e8 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -133,25 +133,11 @@ basic_reboot_apps() -> RebootListPath = filename:join([PrivDir, "reboot_lists.eterm"]), {ok, [ #{ - common_business_apps := CommonBusinessApps0, + common_business_apps := CommonBusinessApps, ee_business_apps := EEBusinessApps, ce_business_apps := CEBusinessApps } ]} = file:consult(RebootListPath), - Filters0 = maps:from_list([ - {App, is_app(App)} - || App <- [quicer, bcrypt, jq, observer] - ]), - CommonBusinessApps = - filter( - CommonBusinessApps0, - %% We don't need to restart these - Filters0#{ - system_monitor => false, - observer => false, - quicer => false - } - ), EditionSpecificApps = case emqx_release:edition() of ee -> EEBusinessApps; @@ -159,25 +145,12 @@ basic_reboot_apps() -> _ -> [] end, BusinessApps = CommonBusinessApps ++ EditionSpecificApps, - ?BASIC_REBOOT_APPS ++ BusinessApps. + ?BASIC_REBOOT_APPS ++ (BusinessApps -- excluded_apps()). -filter(AppList, Filters) -> - lists:foldr( - fun(App, Acc) -> - AppName = - case App of - {Name, _Type} -> Name; - Name when is_atom(Name) -> Name - end, - ShouldKeep = maps:get(AppName, Filters, true), - case ShouldKeep of - true -> [App | Acc]; - false -> Acc - end - end, - [], - AppList - ). +excluded_apps() -> + OptionalApps = [bcrypt, jq, observer], + [system_monitor, observer_cli] ++ + [App || App <- OptionalApps, not is_app(App)]. is_app(Name) -> case application:load(Name) of diff --git a/mix.exs b/mix.exs index 5094ed07c..6836137a9 100644 --- a/mix.exs +++ b/mix.exs @@ -341,16 +341,6 @@ defmodule EMQXUmbrella.MixProject do } ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm") - db_apps = filter(db_apps, %{mnesia_rocksdb: enable_rocksdb?()}) - - common_business_apps = - filter(common_business_apps, %{ - quicer: enable_quicer?(), - bcrypt: enable_bcrypt?(), - jq: enable_jq?(), - observer: is_app?(:observer) - }) - edition_specific_apps = if edition_type == :enterprise do ee_business_apps @@ -360,24 +350,31 @@ defmodule EMQXUmbrella.MixProject do business_apps = common_business_apps ++ edition_specific_apps - Enum.map(system_apps, fn app -> - if is_atom(app), do: {app, :permanent}, else: app - end) ++ - Enum.map(db_apps, &{&1, :load}) ++ - [emqx_machine: :permanent] ++ - Enum.map(business_apps, &{&1, :load}) + excluded_apps = excluded_apps() + + system_apps = + Enum.map(system_apps, fn app -> + if is_atom(app), do: {app, :permanent}, else: app + end) + + db_apps = Enum.map(db_apps, &{&1, :load}) + business_apps = Enum.map(business_apps, &{&1, :load}) + + [system_apps, db_apps, [emqx_machine: :permanent], business_apps] + |> List.flatten() + |> Keyword.reject(fn {app, _type} -> app in excluded_apps end) end - defp filter(apps, filters) do - Enum.filter(apps, fn app -> - app_name = - case app do - {app_name, _type} -> app_name - app_name when is_atom(app_name) -> app_name - end - - Map.get(filters, app_name, true) - end) + defp excluded_apps() do + %{ + mnesia_rocksdb: enable_rocksdb?(), + quicer: enable_quicer?(), + bcrypt: enable_bcrypt?(), + jq: enable_jq?(), + observer: is_app?(:observer) + } + |> Enum.reject(&elem(&1, 1)) + |> Enum.map(&elem(&1, 0)) end defp is_app?(name) do diff --git a/rebar.config.erl b/rebar.config.erl index a326d6fdc..06dd0f6eb 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -388,45 +388,35 @@ overlay_vars_pkg(pkg) -> relx_apps(ReleaseType, Edition) -> {ok, [ #{ - db_apps := DBApps0, + db_apps := DBApps, system_apps := SystemApps, - common_business_apps := CommonBusinessApps0, + common_business_apps := CommonBusinessApps, ee_business_apps := EEBusinessApps, ce_business_apps := CEBusinessApps } ]} = file:consult("apps/emqx_machine/priv/reboot_lists.eterm"), - DBApps = filter(DBApps0, #{mnesia_rocksdb => is_rocksdb_supported()}), - CommonBusinessApps = - filter(CommonBusinessApps0, #{ - quicer => is_quicer_supported(), - bcrypt => provide_bcrypt_release(ReleaseType), - jq => is_jq_supported(), - observer => is_app(observer) - }), EditionSpecificApps = case Edition of ee -> EEBusinessApps; ce -> CEBusinessApps end, BusinessApps = CommonBusinessApps ++ EditionSpecificApps, + ExcludedApps = excluded_apps(ReleaseType), SystemApps ++ %% EMQX starts the DB and the business applications: - [{App, load} || App <- DBApps] ++ + [{App, load} || App <- (DBApps -- ExcludedApps)] ++ [emqx_machine] ++ - [{App, load} || App <- BusinessApps]. + [{App, load} || App <- (BusinessApps -- ExcludedApps)]. -filter(AppList, Filters) -> - lists:filter( - fun(App) -> - AppName = - case App of - {Name, _Type} -> Name; - Name when is_atom(Name) -> Name - end, - maps:get(AppName, Filters, true) - end, - AppList - ). +excluded_apps(ReleaseType) -> + OptionalApps = [ + {quicer, is_quicer_supported()}, + {bcrypt, provide_bcrypt_release(ReleaseType)}, + {jq, is_jq_supported()}, + {observer, is_app(observer)}, + {mnesia_rocksdb, is_rocksdb_supported()} + ], + [App || {App, false} <- OptionalApps]. is_app(Name) -> case application:load(Name) of From 77e2d852e5e9babaa79903090634f5e1669b195b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Jul 2023 09:55:05 -0300 Subject: [PATCH 17/28] fix: add emqx_durable_storage to app list --- apps/emqx_machine/priv/reboot_lists.eterm | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 62298697c..3b821c096 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -39,6 +39,7 @@ %% started by emqx_machine system_monitor, emqx_utils, + emqx_durable_storage, emqx_http_lib, emqx_resource, emqx_connector, From b9f0cd7ba43910af430091a6c7b998c295a5431f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Jul 2023 21:01:28 +0800 Subject: [PATCH 18/28] chore: pretty the cli output style --- apps/emqx_conf/src/emqx_conf_cli.erl | 42 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index f0106fed1..fde3059d3 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -270,25 +270,37 @@ check_res(Node, Key, {ok, _}, _Conf, _Mode) -> emqx_ctl:print("load ~ts on ~p ok~n", [Key, Node]), ok; check_res(_Node, Key, {error, Reason}, Conf, Mode) -> - Warning = io_lib:format("~ts ~ts failed~n~p~n", [Mode, Key, Reason]), - emqx_ctl:warning(Warning, []), - ActiveMsg = "--------active configuration--------\n", - Active = hocon_pp:do(#{Key => emqx_conf:get_raw([Key])}, #{}), - FailedMsg = io_lib:format("--------failed to ~ts with----------~n", [Mode]), - New = hocon_pp:do(#{Key => Conf}, #{}), + Warning = + "Can't ~ts the new configurations!~n" + "Root key: ~ts~n" + "Reason: ~p~n", + emqx_ctl:warning(Warning, [Mode, Key, Reason]), + ActiveMsg0 = + "The effective configurations:~n" + "```~n" + "~ts```~n~n", + ActiveMsg = io_lib:format(ActiveMsg0, [hocon_pp:do(#{Key => emqx_conf:get_raw([Key])}, #{})]), + FailedMsg0 = + "Try to ~ts with:~n" + "```~n" + "~ts```~n", + FailedMsg = io_lib:format(FailedMsg0, [Mode, hocon_pp:do(#{Key => Conf}, #{})]), SuggestMsg = suggest_msg(Mode), - Msg = iolist_to_binary([ActiveMsg, Active, FailedMsg, New, SuggestMsg]), + Msg = iolist_to_binary([ActiveMsg, FailedMsg, SuggestMsg]), emqx_ctl:print("~ts", [Msg]), {error, iolist_to_binary([Warning, Msg])}. -suggest_msg(merge) -> - "Merge conflict with the active Key.\n" - "Suggest to use the replace option to completely replace the configuration,\n" - "instead of merging conflict.\n\n"; -suggest_msg(replace) -> - "Replace failed with an incomplete configuration.\n" - "Suggest to use the merge option to only update sub configuration,\n" - "instead of replacing with an incomplete configuration.\n\n". +suggest_msg(Mode) when Mode == merge orelse Mode == replace -> + RetryMode = + case Mode of + merge -> "replace"; + replace -> "merge" + end, + io_lib:format( + "Tips: There may be some conflicts in the new configuration under `~ts` mode,~n" + "Please retry with the `~ts` mode.~n", + [Mode, RetryMode] + ). check_config(Conf) -> case check_keys_is_not_readonly(Conf) of From 6cd503865bf4cbc5d0b1579cf6c7c91d646623f9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Jul 2023 11:30:40 -0300 Subject: [PATCH 19/28] fix(machine_boot): ensure `emqx_bridge` starts after its companion apps We need to reverse the dependency of `emqx_bridge` and `emqx_bridge_*`, because the former loads and starts bridges during its application startup. If the individual bridge application being loaded has not started with its dependencies, the supervision tree will not be ready for that. --- .../src/emqx_bridge_cassandra.app.src | 1 - .../src/emqx_bridge_clickhouse.app.src | 1 - .../src/emqx_bridge_dynamo.app.src | 1 - .../src/emqx_bridge_gcp_pubsub.app.src | 1 - .../emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 6 +++++ .../src/emqx_bridge_hstreamdb.app.src | 1 - .../src/emqx_bridge_http.app.src | 2 +- .../src/emqx_bridge_influxdb.app.src | 1 - .../src/emqx_bridge_iotdb.app.src | 1 - .../src/emqx_bridge_kafka.app.src | 1 - .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 11 ++++++++-- .../src/emqx_bridge_matrix.app.src | 3 +-- .../src/emqx_bridge_mongodb.app.src | 1 - .../src/emqx_bridge_mqtt.app.src | 3 +-- .../src/emqx_bridge_mysql.app.src | 1 - .../src/emqx_bridge_opents.app.src | 1 - .../src/emqx_bridge_oracle.app.src | 1 - .../src/emqx_bridge_pgsql.app.src | 3 +-- .../src/emqx_bridge_pulsar.app.src | 1 - .../src/emqx_bridge_rabbitmq.app.src | 2 -- .../src/emqx_bridge_redis.app.src | 1 - .../src/emqx_bridge_rocketmq.app.src | 2 +- .../src/emqx_bridge_sqlserver.app.src | 2 +- .../src/emqx_bridge_tdengine.app.src | 1 - .../src/emqx_bridge_timescale.app.src | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 22 ++++++++++++++++++- 26 files changed, 43 insertions(+), 30 deletions(-) diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index f449588cc..de790ab46 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, ecql ]}, {env, []}, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index cfb08f47b..a10361c51 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, clickhouse ]}, {env, []}, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index 824f5ee7b..ed5078432 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, erlcloud ]}, {env, []}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 10722e6ce..53d36361c 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge_http, ehttpc ]}, {env, []}, diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 2a04b5ee1..80ffdfe1a 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -598,6 +598,7 @@ start_cluster(Cluster) -> end, Cluster ), + NumNodes = length(Nodes), on_exit(fun() -> emqx_utils:pmap( fun(N) -> @@ -607,6 +608,11 @@ start_cluster(Cluster) -> Nodes ) end), + {ok, _} = snabbkaffe:block_until( + %% -1 because only those that join the first node will emit the event. + ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}), + 30_000 + ), Nodes. wait_for_cluster_rpc(Node) -> diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 0549dd020..2a800baca 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, hstreamdb_erl ]}, {env, []}, diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src index f8097dec2..859f80f53 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http.app.src +++ b/apps/emqx_bridge_http/src/emqx_bridge_http.app.src @@ -2,7 +2,7 @@ {description, "EMQX HTTP Bridge and Connector Application"}, {vsn, "0.1.1"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, ehttpc]}, + {applications, [kernel, stdlib, emqx_connector, emqx_resource, ehttpc]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 71b95a40d..a612c225b 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, influxdb ]}, {env, []}, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index d26cbe873..b79c4c2ce 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -11,7 +11,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge_http, %% for module emqx_connector_http emqx_connector ]}, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index 87c1841e5..997b768b3 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -7,7 +7,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, telemetry, wolff, brod, diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 74fde6426..f1f2ce362 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1071,13 +1071,14 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]}, + {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_kafka, emqx_bridge]}, {listener_ports, []}, {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, {load_schema, true}, {start_autocluster, true}, {schema_mod, emqx_enterprise_schema}, + {load_apps, [emqx_machine]}, {env_handler, fun (emqx) -> application:set_env(emqx, boot_modules, [broker, router]), @@ -1901,6 +1902,7 @@ t_cluster_node_down(Config) -> ?check_trace( begin {_N2, Opts2} = lists:nth(2, Cluster), + NumNodes = length(Cluster), Nodes = [N1, N2 | _] = lists:map( @@ -1925,6 +1927,11 @@ t_cluster_node_down(Config) -> 15_000 ), wait_for_cluster_rpc(N2), + {ok, _} = snabbkaffe:block_until( + %% -1 because only those that join the first node will emit the event. + ?match_n_events(NumNodes - 1, #{?snk_kind := emqx_machine_boot_apps_started}), + 30_000 + ), erpc:call(N2, fun() -> {ok, _} = create_bridge(Config) end), {ok, _} = snabbkaffe:receive_events(SRef0), lists:foreach( @@ -1980,7 +1987,7 @@ t_cluster_node_down(Config) -> ?assertEqual(NPartitions, map_size(Assignments)), NumPublished = ets:info(TId, size), %% All published messages are eventually received. - Published = receive_published(#{n => NumPublished, timeout => 3_000}), + Published = receive_published(#{n => NumPublished, timeout => 10_000}), ct:pal("published:\n ~p", [Published]), ok end diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src index 42129bfc7..14aca1f75 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -5,8 +5,7 @@ {applications, [ kernel, stdlib, - emqx_resource, - emqx_bridge + emqx_resource ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index fa3ebd3c9..35bcc3fc4 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -7,7 +7,6 @@ stdlib, emqx_connector, emqx_resource, - emqx_bridge, emqx_mongodb ]}, {env, []}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 853e98eb5..052b271f8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,14 +1,13 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, stdlib, emqx, emqx_resource, - emqx_bridge, emqtt ]}, {env, []}, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 2ecdd6a6a..252b8ff00 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -7,7 +7,6 @@ stdlib, emqx_connector, emqx_resource, - emqx_bridge, emqx_mysql ]}, {env, []}, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 6ec938afd..b106bb844 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, opentsdb ]}, {env, []}, diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src index a05533da3..4f46ce464 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, emqx_oracle ]}, {env, []}, diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index ade791a6d..85131baf0 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -5,8 +5,7 @@ {applications, [ kernel, stdlib, - emqx_resource, - emqx_bridge + emqx_resource ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index 99fb25c33..ed468a833 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, pulsar ]}, {env, []}, diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index e9ef4d524..c6c0c3897 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -6,8 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, - ecql, rabbit_common, amqp_client ]}, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index bc21adcad..b380bc86d 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -7,7 +7,6 @@ stdlib, emqx_connector, emqx_resource, - emqx_bridge, emqx_redis ]}, {env, []}, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index e18b98e3a..e158a2e46 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise RocketMQ Bridge"}, {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, emqx_resource, emqx_bridge, rocketmq]}, + {applications, [kernel, stdlib, emqx_resource, rocketmq]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 35f4587b0..3aa8b3b68 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise SQL Server Bridge"}, {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_resource, emqx_bridge, odbc]}, + {applications, [kernel, stdlib, emqx_resource, odbc]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index e4c946162..be57fa880 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, tdengine ]}, {env, []}, diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src index 7a4aeeb56..adb024591 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise TimescaleDB Bridge"}, {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_resource, emqx_bridge]}, + {applications, [kernel, stdlib, emqx_resource]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index cdb2c24e8..82b909b4f 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -160,7 +160,8 @@ is_app(Name) -> end. sorted_reboot_apps() -> - Apps = [{App, app_deps(App)} || App <- reboot_apps()], + Apps0 = [{App, app_deps(App)} || App <- reboot_apps()], + Apps = inject_bridge_deps(Apps0), sorted_reboot_apps(Apps). app_deps(App) -> @@ -169,6 +170,25 @@ app_deps(App) -> {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List) end. +%% `emqx_bridge' is special in that it needs all the bridges apps to +%% be started before it, so that, when it loads the bridges from +%% configuration, the bridge app and its dependencies need to be up. +inject_bridge_deps(RebootAppDeps) -> + BridgeApps = [ + App + || {App, _Deps} <- RebootAppDeps, + lists:prefix("emqx_bridge_", atom_to_list(App)) + ], + lists:map( + fun + ({emqx_bridge, Deps0}) when is_list(Deps0) -> + {emqx_bridge, Deps0 ++ BridgeApps}; + (App) -> + App + end, + RebootAppDeps + ). + sorted_reboot_apps(Apps) -> G = digraph:new(), try From d3e4fda21d0563fd142f2704a02020966998c85f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 21 Jul 2023 12:31:22 +0800 Subject: [PATCH 20/28] fix: hocon_pp quote string for special char --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 2b6e8f172..b1f70fbc4 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.14"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 6836137a9..e7ace90b9 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.13", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.14", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 287b7b0b6..30ffd077d 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.14"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From a1a7c4fab6c8d6a6d63870a55106fb14978e864d Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 18 Jul 2023 20:35:21 +0300 Subject: [PATCH 21/28] fix(emqx_rule_engine): fix typo in `behaviour` module attribute The type resulted in missing `rule_engine` config after importing data from a backup file. Fixes: EMQX-10590 --- apps/emqx_rule_engine/src/emqx_rule_engine.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index f0640c7dc..66c82d3a1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). -behaviour(emqx_config_handler). --behaiour(emqx_config_backup). +-behaviour(emqx_config_backup). -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). From b37920d2e5699d4530c0efe7059450499d7e2824 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 18 Jul 2023 20:39:46 +0300 Subject: [PATCH 22/28] feat: add `topic_metrics` and `slow_subs` configuration to data import/export Fixes: EMQX-10590 --- .../src/emqx_mgmt_data_backup.erl | 3 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_modules/src/emqx_modules_conf.erl | 41 +++++++++++++++++- .../test/emqx_modules_conf_SUITE.erl | 42 +++++++++++++++++-- changes/ce/fix-11322.en.md | 4 ++ 5 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 changes/ce/fix-11322.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index bdb9cf666..b83a46903 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -57,7 +57,8 @@ <<"flapping_detect">>, <<"broker">>, <<"force_gc">>, - <<"zones">> + <<"zones">>, + <<"slow_subs">> ]). -define(DEFAULT_OPTS, #{}). diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 1b934e015..4de1c2e9b 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index 69a69cb12..e5604280d 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -18,6 +18,7 @@ -module(emqx_modules_conf). -behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). %% Load/Unload -export([ @@ -37,6 +38,11 @@ post_config_update/5 ]). +%% Data backup +-export([ + import_config/1 +]). + %%-------------------------------------------------------------------- %% Load/Unload %%-------------------------------------------------------------------- @@ -78,6 +84,20 @@ remove_topic_metrics(Topic) -> {error, Reason} -> {error, Reason} end. +%%-------------------------------------------------------------------- +%% Data backup (Topic-Metrics) +%%-------------------------------------------------------------------- + +import_config(#{<<"topic_metrics">> := Topics}) -> + case emqx_conf:update([topic_metrics], {merge_topics, Topics}, #{override_to => cluster}) of + {ok, _} -> + {ok, #{root_key => topic_metrics, changed => []}}; + Error -> + {error, #{root_key => topic_metrics, reason => Error}} + end; +import_config(_RawConf) -> + {ok, #{root_key => topic_metrics, changed => []}}. + %%-------------------------------------------------------------------- %% Config Handler %%-------------------------------------------------------------------- @@ -103,7 +123,13 @@ pre_config_update(_, {remove_topic_metrics, Topic0}, RawConf) -> {ok, RawConf -- [Topic]}; _ -> {error, not_found} - end. + end; +pre_config_update(_, {merge_topics, NewConf}, OldConf) -> + KeyFun = fun(#{<<"topic">> := T}) -> T end, + MergedConf = emqx_utils:merge_lists(OldConf, NewConf, KeyFun), + {ok, MergedConf}; +pre_config_update(_, NewConf, _OldConf) -> + {ok, NewConf}. -spec post_config_update( list(atom()), @@ -113,7 +139,6 @@ pre_config_update(_, {remove_topic_metrics, Topic0}, RawConf) -> emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. - post_config_update( _, {add_topic_metrics, Topic}, @@ -135,6 +160,18 @@ post_config_update( case emqx_topic_metrics:deregister(Topic) of ok -> ok; {error, Reason} -> {error, Reason} + end; +post_config_update(_, _UpdateReq, NewConfig, OldConfig, _AppEnvs) -> + #{ + removed := Removed, + added := Added + } = emqx_utils:diff_lists(NewConfig, OldConfig, fun(#{topic := T}) -> T end), + Deregistered = [emqx_topic_metrics:deregister(T) || #{topic := T} <- Removed], + Registered = [emqx_topic_metrics:register(T) || #{topic := T} <- Added], + Errs = [Res || Res <- Registered ++ Deregistered, Res =/= ok], + case Errs of + [] -> ok; + _ -> {error, Errs} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl index 14e477bf9..b95cc2fe3 100644 --- a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl +++ b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl @@ -39,12 +39,46 @@ end_per_suite(_Conf) -> init_per_testcase(_CaseName, Conf) -> Conf. +end_per_testcase(_CaseName, _Conf) -> + [emqx_modules_conf:remove_topic_metrics(T) || T <- emqx_modules_conf:topic_metrics()], + ok. + %%-------------------------------------------------------------------- %% Cases %%-------------------------------------------------------------------- -t_topic_metrics_list(_) -> - ok. - t_topic_metrics_add_remove(_) -> - ok. + ?assertEqual([], emqx_modules_conf:topic_metrics()), + ?assertMatch({ok, _}, emqx_modules_conf:add_topic_metrics(<<"test-topic">>)), + ?assertEqual([<<"test-topic">>], emqx_modules_conf:topic_metrics()), + ?assertEqual(ok, emqx_modules_conf:remove_topic_metrics(<<"test-topic">>)), + ?assertEqual([], emqx_modules_conf:topic_metrics()), + ?assertMatch({error, _}, emqx_modules_conf:remove_topic_metrics(<<"test-topic">>)). + +t_topic_metrics_merge_update(_) -> + ?assertEqual([], emqx_modules_conf:topic_metrics()), + ?assertMatch({ok, _}, emqx_modules_conf:add_topic_metrics(<<"test-topic-before-import1">>)), + ?assertMatch({ok, _}, emqx_modules_conf:add_topic_metrics(<<"test-topic-before-import2">>)), + ImportConf = #{ + <<"topic_metrics">> => + [ + #{<<"topic">> => <<"imported_topic1">>}, + #{<<"topic">> => <<"imported_topic2">>} + ] + }, + ?assertMatch({ok, _}, emqx_modules_conf:import_config(ImportConf)), + ExpTopics = [ + <<"test-topic-before-import1">>, + <<"test-topic-before-import2">>, + <<"imported_topic1">>, + <<"imported_topic2">> + ], + ?assertEqual(ExpTopics, emqx_modules_conf:topic_metrics()). + +t_topic_metrics_update(_) -> + ?assertEqual([], emqx_modules_conf:topic_metrics()), + ?assertMatch({ok, _}, emqx_modules_conf:add_topic_metrics(<<"test-topic-before-update1">>)), + ?assertMatch({ok, _}, emqx_modules_conf:add_topic_metrics(<<"test-topic-before-update2">>)), + UpdConf = [#{<<"topic">> => <<"new_topic1">>}, #{<<"topic">> => <<"new_topic2">>}], + ?assertMatch({ok, _}, emqx_conf:update([topic_metrics], UpdConf, #{override_to => cluster})), + ?assertEqual([<<"new_topic1">>, <<"new_topic2">>], emqx_modules_conf:topic_metrics()). diff --git a/changes/ce/fix-11322.en.md b/changes/ce/fix-11322.en.md new file mode 100644 index 000000000..fca72c01a --- /dev/null +++ b/changes/ce/fix-11322.en.md @@ -0,0 +1,4 @@ +Import additional configurations from EMQX backup file (`emqx ctl import` command): + - rule_engine (previously not imported due to the bug) + - topic_metrics (previously not implemented) + - slow_subs (previously not implemented). From 520ae0e9d3d39718e32e6775bd75741a6731e897 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 21 Jul 2023 15:51:21 +0800 Subject: [PATCH 23/28] chore: remove zone desc from document --- apps/emqx/src/emqx_schema.erl | 4 +--- rel/i18n/emqx_mgmt_api_configs.hocon | 8 ++++---- rel/i18n/emqx_mgmt_api_key_schema.hocon | 11 +++++------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d2edbcf2d..bd0c87974 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1836,9 +1836,7 @@ desc("stats") -> desc("authorization") -> "Settings for client authorization."; desc("mqtt") -> - "Global MQTT configuration.
" - "The configs here work as default values which can be overridden\n" - "in zone configs"; + "Global MQTT configuration."; desc("authz_cache") -> "Settings for the authorization cache."; desc("zone") -> diff --git a/rel/i18n/emqx_mgmt_api_configs.hocon b/rel/i18n/emqx_mgmt_api_configs.hocon index 47852349e..44d461081 100644 --- a/rel/i18n/emqx_mgmt_api_configs.hocon +++ b/rel/i18n/emqx_mgmt_api_configs.hocon @@ -23,14 +23,14 @@ rest_conf_query.label: """Reset the config entry with query""" get_global_zone_configs.desc: -"""Get the global zone configs""" +"""Get the MQTT-related configuration""" get_global_zone_configs.label: -"""Get the global zone configs""" +"""Get the MQTT-related configuration""" update_global_zone_configs.desc: -"""Update global zone configs""" +"""Update MQTT-related configuration""" update_global_zone_configs.label: -"""Update global zone configs""" +"""Update MQTT-related configuration""" get_node_level_limiter_configs.desc: """Get the node-level limiter configs""" diff --git a/rel/i18n/emqx_mgmt_api_key_schema.hocon b/rel/i18n/emqx_mgmt_api_key_schema.hocon index 0dc11c7ac..c217dc2db 100644 --- a/rel/i18n/emqx_mgmt_api_key_schema.hocon +++ b/rel/i18n/emqx_mgmt_api_key_schema.hocon @@ -7,12 +7,11 @@ api_key.label: """API Key""" bootstrap_file.desc: -"""Bootstrap file is used to add an api_key when emqx is launched, - the format is: - ``` - 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK - ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL - ```""" +"""The bootstrap file provides API keys for EMQX. +EMQX will load these keys on startup to authorize API requests. +It contains key-value pairs in the format:`api_key:api_secret`. +Each line specifies an API key and its associated secret. +""" bootstrap_file.label: """Initialize api_key file.""" From c73bbd1d8fd66abdc75cbaa5dd93efff7a1d03ab Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:45:21 +0200 Subject: [PATCH 24/28] chore(changelog): Add changelog for 11309 --- changes/ce/fix-11309.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-11309.en.md diff --git a/changes/ce/fix-11309.en.md b/changes/ce/fix-11309.en.md new file mode 100644 index 000000000..5f028be55 --- /dev/null +++ b/changes/ce/fix-11309.en.md @@ -0,0 +1,2 @@ +Improve startup order of EMQX applications. +Simplify build scripts and improve code reuse. From ee97cf8e33fc7ccaea76027bc7ba59a5323d4762 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 21 Jul 2023 12:17:47 +0300 Subject: [PATCH 25/28] fix(emqx_modules): ignore already_existed/not_found errors for added/removed topic metrics --- apps/emqx_modules/src/emqx_modules_conf.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index e5604280d..ebe9b83df 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -168,10 +168,11 @@ post_config_update(_, _UpdateReq, NewConfig, OldConfig, _AppEnvs) -> } = emqx_utils:diff_lists(NewConfig, OldConfig, fun(#{topic := T}) -> T end), Deregistered = [emqx_topic_metrics:deregister(T) || #{topic := T} <- Removed], Registered = [emqx_topic_metrics:register(T) || #{topic := T} <- Added], - Errs = [Res || Res <- Registered ++ Deregistered, Res =/= ok], - case Errs of + DeregisteredErrs = [Res || Res <- Deregistered, Res =/= ok, Res =/= {error, topic_not_found}], + RegisteredErrs = [Res || Res <- Registered, Res =/= ok, Res =/= {error, already_existed}], + case DeregisteredErrs ++ RegisteredErrs of [] -> ok; - _ -> {error, Errs} + Errs -> {error, Errs} end. %%-------------------------------------------------------------------- From 3a3b843f4a781e65c183ccbe7f1f93514c20ee55 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 21 Jul 2023 13:16:54 +0200 Subject: [PATCH 26/28] fix(machine): add emqx_bridge_kinesis to reboot_lists.eterm --- apps/emqx_machine/priv/reboot_lists.eterm | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 3b821c096..500a47d8f 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -97,6 +97,7 @@ emqx_bridge_tdengine, emqx_bridge_timescale, emqx_bridge_sqlserver, + emqx_bridge_kinesis, emqx_oracle, emqx_bridge_oracle, emqx_bridge_rabbitmq, From c2c3da073f3fb22e3cefa3d7a9f8917edc3c9f80 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 21 Jul 2023 08:46:51 +0200 Subject: [PATCH 27/28] chore: v5.1.2 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 2412de99e..09b435e6e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.1.1"). +-define(EMQX_RELEASE_CE, "5.1.2"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.1.1-alpha.2"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index a2262da8b..67166708b 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.1.1 +version: 5.1.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.1.1 +appVersion: 5.1.2 From 7df0c6a808ee9a546fd0bfddbdc1a5b5512fdd1b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 21 Jul 2023 13:26:55 +0200 Subject: [PATCH 28/28] docs: Generate changelog for v5.1.2 --- changes/v5.1.2.en.md | 72 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 changes/v5.1.2.en.md diff --git a/changes/v5.1.2.en.md b/changes/v5.1.2.en.md new file mode 100644 index 000000000..a6b7d127b --- /dev/null +++ b/changes/v5.1.2.en.md @@ -0,0 +1,72 @@ +# v5.1.2 + +## Enhancements + +- [#11124](https://github.com/emqx/emqx/pull/11124) Release packages for Amazon Linux 2023 + +- [#11226](https://github.com/emqx/emqx/pull/11226) Unify the listener switch to `enable`, while being compatible with the previous `enabled`. + +- [#11249](https://github.com/emqx/emqx/pull/11249) Support HTTP API for setting alarm watermark of license. + +- [#11251](https://github.com/emqx/emqx/pull/11251) Add `/cluster/topology` HTTP API endpoint + + `GET` request to the endpoint returns the cluster topology: connections between RLOG core and replicant nodes. + +- [#11253](https://github.com/emqx/emqx/pull/11253) The Webhook/HTTP bridge has been refactored to its own Erlang application. This allows for more flexibility in the future, and also allows for the bridge to be run as a standalone application. + +- [#11289](https://github.com/emqx/emqx/pull/11289) Release packages for Debian 12. + +- [#11290](https://github.com/emqx/emqx/pull/11290) Updated `jq` dependency to version 0.3.10 which includes `oniguruma` library update to version 6.9.8 with few minor security fixes. + +- [#11291](https://github.com/emqx/emqx/pull/11291) Updated RocksDB version to 1.8.0-emqx-1 via ekka update to 0.15.6. + +- [#11236](https://github.com/emqx/emqx/pull/11236) Improve the speed of clients querying in HTTP API `/clients` endpoint with default parameters + +## Bug Fixes + +- [#11065](https://github.com/emqx/emqx/pull/11065) Avoid logging irrelevant error messages during EMQX shutdown. + +- [#11077](https://github.com/emqx/emqx/pull/11077) Fixes crash when updating binding with a non-integer port. + +- [#11184](https://github.com/emqx/emqx/pull/11184) Config value for `max_packet_size` has a max value of 256MB defined by protocol. This is now enforced and any configuration with a value greater than that will break. + +- [#11192](https://github.com/emqx/emqx/pull/11192) Fix produces valid HOCON file when atom type is used. + Remove unnecessary `"` from HOCON file. + +- [#11195](https://github.com/emqx/emqx/pull/11195) Avoid to create duplicated subscription by HTTP API or client in Stomp gateway + +- [#11206](https://github.com/emqx/emqx/pull/11206) Make the username and password params of CoAP client to optional in connection mode. + +- [#11208](https://github.com/emqx/emqx/pull/11208) Fix the issue of abnormal data statistics for LwM2M client. + +- [#11211](https://github.com/emqx/emqx/pull/11211) Consistently return `404` for `DELETE` operations on non-existent resources. + +- [#11214](https://github.com/emqx/emqx/pull/11214) Fix a bug where node configuration may fail to synchronize correctly when joining the cluster. + +- [#11229](https://github.com/emqx/emqx/pull/11229) Fixed an issue preventing plugins from starting/stopping after changing configuration via `emqx ctl conf load`. + +- [#11237](https://github.com/emqx/emqx/pull/11237) The `headers` default value in /prometheus API should be a map instead of a list. + +- [#11250](https://github.com/emqx/emqx/pull/11250) Fix while a WebSocket packet contains more than one MQTT packet, the order of MQTT packets will be reversed. + + +- [#11271](https://github.com/emqx/emqx/pull/11271) Ensure that the range of percentage type is from 0% to 100%. + +- [#11272](https://github.com/emqx/emqx/pull/11272) Fix a typo in the log, when EMQX received an abnormal `PUBREL` packet, the `pubrel` was mistakenly typo as `pubrec`. + +- [#11281](https://github.com/emqx/emqx/pull/11281) Restored support for the special `$queue/` shared subscription. + +- [#11294](https://github.com/emqx/emqx/pull/11294) Fix `emqx_ctl cluster join`, `leave`, and `status` commands. + +- [#11296](https://github.com/emqx/emqx/pull/11296) Import additional configurations from EMQX backup file (`emqx ctl import` command): + - rule_engine (previously not imported due to the bug) + - topic_metrics (previously not implemented) + - slow_subs (previously not implemented). + +- [#11309](https://github.com/emqx/emqx/pull/11309) Improve startup order of EMQX applications. + Simplify build scripts and improve code reuse. + +- [#11322](https://github.com/emqx/emqx/pull/11322) Import additional configurations from EMQX backup file (`emqx ctl import` command): + - rule_engine (previously not imported due to the bug) + - topic_metrics (previously not implemented) + - slow_subs (previously not implemented).