From 13f50b2ba9e92d77c233d453f87d93192e43d07c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 21:05:10 +0800 Subject: [PATCH 001/194] chore: update README files --- apps/emqx_conf/README.md | 16 ++++++++++++++++ apps/emqx_plugins/README.md | 12 ++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 apps/emqx_conf/README.md create mode 100644 apps/emqx_plugins/README.md diff --git a/apps/emqx_conf/README.md b/apps/emqx_conf/README.md new file mode 100644 index 000000000..726fd72cd --- /dev/null +++ b/apps/emqx_conf/README.md @@ -0,0 +1,16 @@ +# Configuration Management + +This application provides configuration management capabilities for EMQX. + +This includes, during compilation: +- Read all configuration schemas and generate the following files: + * `config-en.md`: documentation for all configuration options. + * `schema-en.json`: JSON description of all configuration schema options. + * `emqx.conf.example`: an example of a complete configuration file. + +At runtime, it provides: +- Cluster configuration synchronization capability. + Responsible for synchronizing hot-update configurations from the HTTP API to the entire cluster + and ensuring consistency. + +In addition, this application manages system-level configurations such as `cluster`, `node`, `log`. diff --git a/apps/emqx_plugins/README.md b/apps/emqx_plugins/README.md new file mode 100644 index 000000000..9c8faccd1 --- /dev/null +++ b/apps/emqx_plugins/README.md @@ -0,0 +1,12 @@ +# Plugins Management + +This application provides the feature for users to upload and install custom, Erlang-based plugins. + +More introduction about [Plugins](https://www.emqx.io/docs/en/v5.0/extensions/plugins.html#develop-emqx-plugins) + +See HTTP API to learn how to [Install/Uninstall a Plugin](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Plugins) + +## Plugin Template + +We provide a [plugin template](https://github.com/emqx/emqx-plugin-template) that +you can use to learn how to write and package custom plugins. From 6e12abff39189f4cb1eb82a7ac32b1620a64dda4 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 20 Apr 2023 17:58:12 +0800 Subject: [PATCH 002/194] fix(rocketmq): allow setting multiple addresses in RocketMQ bridge --- .../src/emqx_ee_connector_rocketmq.erl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 205359bb8..389e1e366 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -38,7 +38,7 @@ roots() -> fields(config) -> [ - {server, server()}, + {servers, servers()}, {topic, mk( binary(), @@ -75,7 +75,7 @@ add_default_fn(OrigFn, Default) -> (Field) -> OrigFn(Field) end. -server() -> +servers() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?ROCKETMQ_HOST_OPTIONS). @@ -97,7 +97,7 @@ is_buffer_supported() -> false. on_start( InstanceId, - #{server := Server, topic := Topic} = Config1 + #{servers := BinServers, topic := Topic} = Config1 ) -> ?SLOG(info, #{ msg => "starting_rocketmq_connector", @@ -105,9 +105,8 @@ on_start( config => redact(Config1) }), Config = maps:merge(default_security_info(), Config1), - {Host, Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS), + Servers = emqx_schema:parse_servers(BinServers, ?ROCKETMQ_HOST_OPTIONS), - Server1 = [{Host, Port}], ClientId = client_id(InstanceId), ClientCfg = #{acl_info => #{}}, @@ -124,7 +123,7 @@ on_start( producers_opts => ProducerOpts }, - case rocketmq:ensure_supervised_client(ClientId, Server1, ClientCfg) of + case rocketmq:ensure_supervised_client(ClientId, Servers, ClientCfg) of {ok, _Pid} -> {ok, State}; {error, _Reason} = Error -> From d865998a63a6bf481148b00751d9b28a42f1d52e Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Apr 2023 11:02:14 +0800 Subject: [PATCH 003/194] fix(rocketmq): fix test cases --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index 0cb14e5c3..33a83d2d8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -136,7 +136,7 @@ rocketmq_config(BridgeType, Config) -> io_lib:format( "bridges.~s.~s {\n" " enable = true\n" - " server = ~p\n" + " servers = ~p\n" " topic = ~p\n" " resource_opts = {\n" " request_timeout = 1500ms\n" From f602900a53fb547049f4c51b404be813d682f8be Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Apr 2023 13:35:29 +0800 Subject: [PATCH 004/194] fix(rocketmq): fix that the status check of RocketMQ bridge may not accurate --- .../emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 205359bb8..67f2d2562 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -154,12 +154,15 @@ on_batch_query(_InstanceId, Query, _State) -> on_get_status(_InstanceId, #{client_id := ClientId}) -> case rocketmq_client_sup:find_client(ClientId) of - {ok, _Pid} -> - connected; + {ok, Pid} -> + status_result(rocketmq_client:get_status(Pid)); _ -> connecting end. +status_result(_Status = true) -> connected; +status_result(_Status) -> connecting. + %%======================================================================================== %% Helper fns %%======================================================================================== From 7704995279935c99f74bbbbfa2e66e5d47df9262 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Apr 2023 15:10:25 +0800 Subject: [PATCH 005/194] fix(rocketmq): expose the driver parameter `sync_timeout` into the RocketMQ bridge configuration --- .../src/emqx_ee_connector_rocketmq.erl | 15 +++++++++++---- rel/i18n/emqx_ee_connector_rocketmq.hocon | 13 ++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 285fa98b4..29f8ef84d 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -44,6 +44,11 @@ fields(config) -> binary(), #{default => <<"TopicTest">>, desc => ?DESC(topic)} )}, + {sync_timeout, + mk( + emqx_schema:duration(), + #{default => <<"3s">>, desc => ?DESC(sync_timeout)} + )}, {refresh_interval, mk( emqx_schema:duration(), @@ -76,7 +81,7 @@ add_default_fn(OrigFn, Default) -> end. servers() -> - Meta = #{desc => ?DESC("server")}, + Meta = #{desc => ?DESC("servers")}, emqx_schema:servers_sc(Meta, ?ROCKETMQ_HOST_OPTIONS). relational_fields() -> @@ -97,7 +102,7 @@ is_buffer_supported() -> false. on_start( InstanceId, - #{servers := BinServers, topic := Topic} = Config1 + #{servers := BinServers, topic := Topic, sync_timeout := SyncTimeout} = Config1 ) -> ?SLOG(info, #{ msg => "starting_rocketmq_connector", @@ -116,8 +121,9 @@ on_start( ProducersMapPID = create_producers_map(ClientId), State = #{ client_id => ClientId, + topic => Topic, topic_tokens => TopicTks, - config => Config, + sync_timeout => SyncTimeout, templates => Templates, producers_map_pid => ProducersMapPID, producers_opts => ProducerOpts @@ -173,9 +179,10 @@ do_query( #{ templates := Templates, client_id := ClientId, + topic := RawTopic, topic_tokens := TopicTks, producers_opts := ProducerOpts, - config := #{topic := RawTopic, resource_opts := #{request_timeout := RequestTimeout}} + sync_timeout := RequestTimeout } = State ) -> ?TRACE( diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index 44dda7931..7f786898e 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -1,6 +1,6 @@ emqx_ee_connector_rocketmq { - server { + servers { desc { en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
@@ -26,6 +26,17 @@ The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" } } + sync_timeout { + desc { + en: """Timeout of RocketMQ driver synchronous call.""" + zh: """RocketMQ 驱动同步调用的超时时间。""" + } + label: { + en: "Sync Timeout" + zh: "同步调用超时时间" + } + } + refresh_interval { desc { en: """RocketMQ Topic Route Refresh Interval.""" From c1000ccaedfc222e71c64c2c20befb6588f9b716 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 19 Apr 2023 16:42:54 +0800 Subject: [PATCH 006/194] fix: always check authn_http's header and ssl_option --- apps/emqx_authn/src/emqx_authn_schema.erl | 26 +++++ .../src/simple_authn/emqx_authn_http.erl | 52 ++++++--- .../test/emqx_authn_https_SUITE.erl | 17 +++ .../emqx_conf/test/emqx_conf_schema_tests.erl | 107 ++++++++++++++++++ 4 files changed, 186 insertions(+), 16 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 112ea2076..a7cdaac5f 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -18,10 +18,12 @@ -elvis([{elvis_style, invalid_dynamic_call, disable}]). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_authn.hrl"). -export([ common_fields/0, roots/0, + validations/0, tags/0, fields/1, authenticator_type/0, @@ -207,3 +209,27 @@ array(Name) -> array(Name, DescId) -> {Name, ?HOCON(?R_REF(Name), #{desc => ?DESC(DescId)})}. + +validations() -> + [ + {check_http_ssl_opts, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_ssl_opts/1, + validation(Conf, CheckFun) + end}, + {check_http_headers, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_headers/1, + validation(Conf, CheckFun) + end} + ]. + +validation(Conf, CheckFun) when is_map(Conf) -> + validation(hocon_maps:get(?CONF_NS, Conf), CheckFun); +validation(undefined, _) -> + ok; +validation([], _) -> + ok; +validation([AuthN | Tail], CheckFun) -> + case CheckFun(#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => AuthN}) of + ok -> validation(Tail, CheckFun); + Error -> Error + end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 3c34d878e..562cfdf6f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -38,6 +38,8 @@ headers/1 ]). +-export([check_headers/1, check_ssl_opts/1]). + -export([ refs/0, union_member_selector/1, @@ -106,8 +108,8 @@ common_fields() -> validations() -> [ - {check_ssl_opts, fun check_ssl_opts/1}, - {check_headers, fun check_headers/1} + {check_ssl_opts, fun ?MODULE:check_ssl_opts/1}, + {check_headers, fun ?MODULE:check_headers/1} ]. url(type) -> binary(); @@ -261,21 +263,39 @@ transform_header_name(Headers) -> ). check_ssl_opts(Conf) -> - {BaseUrl, _Path, _Query} = parse_url(get_conf_val("url", Conf)), - case BaseUrl of - <<"https://", _/binary>> -> - case get_conf_val("ssl.enable", Conf) of - true -> ok; - false -> false - end; - <<"http://", _/binary>> -> - ok + case get_conf_val("url", Conf) of + undefined -> + ok; + Url -> + {BaseUrl, _Path, _Query} = parse_url(Url), + case BaseUrl of + <<"https://", _/binary>> -> + case get_conf_val("ssl.enable", Conf) of + true -> + ok; + false -> + <<"it's required to enable the TLS option to establish a https connection">> + end; + <<"http://", _/binary>> -> + ok + end end. check_headers(Conf) -> - Method = to_bin(get_conf_val("method", Conf)), - Headers = get_conf_val("headers", Conf), - Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). + case get_conf_val("headers", Conf) of + undefined -> + ok; + Headers -> + case to_bin(get_conf_val("method", Conf)) of + <<"post">> -> + ok; + <<"get">> -> + case maps:is_key(<<"content-type">>, Headers) of + false -> ok; + true -> <<"HTTP GET requests cannot include content-type header.">> + end + end + end. parse_url(Url) -> case string:split(Url, "//", leading) of @@ -310,7 +330,7 @@ parse_config( method => Method, path => Path, headers => ensure_header_name_type(Headers), - base_path_templete => emqx_authn_utils:parse_str(Path), + base_path_template => emqx_authn_utils:parse_str(Path), base_query_template => emqx_authn_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)) ), @@ -323,7 +343,7 @@ parse_config( generate_request(Credential, #{ method := Method, headers := Headers0, - base_path_templete := BasePathTemplate, + base_path_template := BasePathTemplate, base_query_template := BaseQueryTemplate, body_template := BodyTemplate }) -> diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index c4315b69f..f23a160d1 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -114,6 +114,22 @@ t_create_invalid_version(_Config) -> emqx_access_control:authenticate(?CREDENTIALS) ). +t_create_disable_ssl_opts_when_https(_Config) -> + {ok, _} = create_https_auth_with_ssl_opts( + #{ + <<"server_name_indication">> => <<"authn-server">>, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>], + <<"enable">> => <<"false">> + } + ), + + ?assertEqual( + {error, not_authorized}, + emqx_access_control:authenticate(?CREDENTIALS) + ). + t_create_invalid_ciphers(_Config) -> {ok, _} = create_https_auth_with_ssl_opts( #{ @@ -135,6 +151,7 @@ t_create_invalid_ciphers(_Config) -> create_https_auth_with_ssl_opts(SpecificSSLOpts) -> AuthConfig = raw_https_auth_config(SpecificSSLOpts), + ct:pal("111:~p~n", [AuthConfig]), emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, AuthConfig}). raw_https_auth_config(SpecificSSLOpts) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 3653b9d19..d6722299f 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -6,6 +6,113 @@ -include_lib("eunit/include/eunit.hrl"). +-define(BASE_CONF, + "" + "\n" + " node {\n" + " name = \"emqx1@127.0.0.1\"\n" + " cookie = \"emqxsecretcookie\"\n" + " data_dir = \"data\"\n" + " }\n" + " cluster {\n" + " name = emqxcl\n" + " discovery_strategy = static\n" + " static.seeds = ~p\n" + " core_nodes = ~p\n" + " }\n" + "" +). +array_nodes_test() -> + ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], + lists:foreach( + fun(Nodes) -> + ConfFile = iolist_to_binary(io_lib:format(?BASE_CONF, [Nodes, Nodes])), + {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), + ClusterDiscovery = proplists:get_value( + cluster_discovery, proplists:get_value(ekka, ConfList) + ), + ?assertEqual( + {static, [{seeds, ExpectNodes}]}, + ClusterDiscovery, + Nodes + ), + ?assertEqual( + ExpectNodes, + proplists:get_value(core_nodes, proplists:get_value(mria, ConfList)), + Nodes + ) + end, + [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] + ), + ok. + +authn_validations_test() -> + BaseConf = iolist_to_binary(io_lib:format(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"])), + DisableSSLWithHttps = + "" + "\n" + "authentication = [\n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"post\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"https://127.0.0.1:8080\"\n" + "}\n" + "]\n" + "", + Conf = <>, + {ok, ConfMap} = hocon:binary(Conf, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_ssl_opts + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap) + ), + BadHeader = + "" + "\n" + "authentication = [\n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"get\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"http://127.0.0.1:8080\"\n" + "}\n" + "]\n" + "", + Conf1 = <>, + {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_headers + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap1) + ), + ok. + doc_gen_test() -> %% the json file too large to encode. { From 397e28f5a4cbd692ffe6b0bb95dd89a401988d65 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 19 Apr 2023 17:10:07 +0800 Subject: [PATCH 007/194] chore: add changlog for authn_http validation --- apps/emqx_authn/test/emqx_authn_https_SUITE.erl | 17 ----------------- changes/ce/fix-10449.en.md | 2 ++ 2 files changed, 2 insertions(+), 17 deletions(-) create mode 100644 changes/ce/fix-10449.en.md diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index f23a160d1..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -114,22 +114,6 @@ t_create_invalid_version(_Config) -> emqx_access_control:authenticate(?CREDENTIALS) ). -t_create_disable_ssl_opts_when_https(_Config) -> - {ok, _} = create_https_auth_with_ssl_opts( - #{ - <<"server_name_indication">> => <<"authn-server">>, - <<"verify">> => <<"verify_peer">>, - <<"versions">> => [<<"tlsv1.2">>], - <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>], - <<"enable">> => <<"false">> - } - ), - - ?assertEqual( - {error, not_authorized}, - emqx_access_control:authenticate(?CREDENTIALS) - ). - t_create_invalid_ciphers(_Config) -> {ok, _} = create_https_auth_with_ssl_opts( #{ @@ -151,7 +135,6 @@ t_create_invalid_ciphers(_Config) -> create_https_auth_with_ssl_opts(SpecificSSLOpts) -> AuthConfig = raw_https_auth_config(SpecificSSLOpts), - ct:pal("111:~p~n", [AuthConfig]), emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, AuthConfig}). raw_https_auth_config(SpecificSSLOpts) -> diff --git a/changes/ce/fix-10449.en.md b/changes/ce/fix-10449.en.md new file mode 100644 index 000000000..e10b52fb4 --- /dev/null +++ b/changes/ce/fix-10449.en.md @@ -0,0 +1,2 @@ +Validate the ssl_options and header configurations when creating authentication http (`authn_http`). +Prior to this, incorrect ssl_options configuration could result in successful creation but the entire authn being unusable. From dc92b4f63f793e866374ce59730431d88d2430f6 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 20 Apr 2023 18:20:51 +0800 Subject: [PATCH 008/194] test: add a test for authn {} --- .../emqx_conf/test/emqx_conf_schema_tests.erl | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index d6722299f..fd2bd4dcd 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -111,6 +111,37 @@ authn_validations_test() -> ]}, hocon_tconf:generate(emqx_conf_schema, ConfMap1) ), + BadHeader2 = + "" + "\n" + "authentication = \n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"get\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"http://127.0.0.1:8080\"\n" + "}\n" + "\n" + "", + Conf2 = <>, + {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_headers + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap2) + ), ok. doc_gen_test() -> From f831a0b8277ce9c6cec25679fe41057bd82d09d2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 21 Apr 2023 11:21:05 +0800 Subject: [PATCH 009/194] chore: update changes/ce/fix-10449.en.md Co-authored-by: JianBo He --- changes/ce/fix-10449.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/fix-10449.en.md b/changes/ce/fix-10449.en.md index e10b52fb4..005dea73c 100644 --- a/changes/ce/fix-10449.en.md +++ b/changes/ce/fix-10449.en.md @@ -1,2 +1,2 @@ Validate the ssl_options and header configurations when creating authentication http (`authn_http`). -Prior to this, incorrect ssl_options configuration could result in successful creation but the entire authn being unusable. +Prior to this, incorrect `ssl` configuration could result in successful creation but the entire authn being unusable. From 1db38de71f44693b3294ed61797b7335f12e06bf Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 21 Apr 2023 11:54:11 +0800 Subject: [PATCH 010/194] chore: apply review suggestions --- .../src/simple_authn/emqx_authn_http.erl | 28 ++- .../emqx_conf/test/emqx_conf_schema_tests.erl | 185 ++++++++---------- apps/emqx_prometheus/TODO | 2 - 3 files changed, 98 insertions(+), 117 deletions(-) delete mode 100644 apps/emqx_prometheus/TODO diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 562cfdf6f..43701cbc7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -263,10 +263,9 @@ transform_header_name(Headers) -> ). check_ssl_opts(Conf) -> - case get_conf_val("url", Conf) of - undefined -> - ok; - Url -> + case is_backend_http(Conf) of + true -> + Url = get_conf_val("url", Conf), {BaseUrl, _Path, _Query} = parse_url(Url), case BaseUrl of <<"https://", _/binary>> -> @@ -278,14 +277,15 @@ check_ssl_opts(Conf) -> end; <<"http://", _/binary>> -> ok - end + end; + false -> + ok end. check_headers(Conf) -> - case get_conf_val("headers", Conf) of - undefined -> - ok; - Headers -> + case is_backend_http(Conf) of + true -> + Headers = get_conf_val("headers", Conf), case to_bin(get_conf_val("method", Conf)) of <<"post">> -> ok; @@ -294,7 +294,15 @@ check_headers(Conf) -> false -> ok; true -> <<"HTTP GET requests cannot include content-type header.">> end - end + end; + false -> + ok + end. + +is_backend_http(Conf) -> + case get_conf_val("backend", Conf) of + http -> true; + _ -> false end. parse_url(Url) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index fd2bd4dcd..667d1766f 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -6,27 +6,27 @@ -include_lib("eunit/include/eunit.hrl"). +%% erlfmt-ignore -define(BASE_CONF, - "" - "\n" - " node {\n" - " name = \"emqx1@127.0.0.1\"\n" - " cookie = \"emqxsecretcookie\"\n" - " data_dir = \"data\"\n" - " }\n" - " cluster {\n" - " name = emqxcl\n" - " discovery_strategy = static\n" - " static.seeds = ~p\n" - " core_nodes = ~p\n" - " }\n" - "" -). + """ + node { + name = \"emqx1@127.0.0.1\" + cookie = \"emqxsecretcookie\" + data_dir = \"data\" + } + cluster { + name = emqxcl + discovery_strategy = static + static.seeds = ~p + core_nodes = ~p + } + """). + array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> - ConfFile = iolist_to_binary(io_lib:format(?BASE_CONF, [Nodes, Nodes])), + ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), ClusterDiscovery = proplists:get_value( @@ -47,101 +47,73 @@ array_nodes_test() -> ), ok. +%% erlfmt-ignore +-define(BASE_AUTHN_ARRAY, + """ + authentication = [ + {backend = \"http\" + body {password = \"${password}\", username = \"${username}\"} + connect_timeout = \"15s\" + enable_pipelining = 100 + headers {\"content-type\" = \"application/json\"} + mechanism = \"password_based\" + method = \"~p\" + pool_size = 8 + request_timeout = \"5s\" + ssl {enable = ~p, verify = \"verify_peer\"} + url = \"~ts\" + } + ] + """ +). + +-define(ERROR(Reason), + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := Reason + } + ]} +). + authn_validations_test() -> - BaseConf = iolist_to_binary(io_lib:format(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"])), - DisableSSLWithHttps = - "" - "\n" - "authentication = [\n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"post\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"https://127.0.0.1:8080\"\n" - "}\n" - "]\n" - "", - Conf = <>, - {ok, ConfMap} = hocon:binary(Conf, #{format => richmap}), - ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_ssl_opts - } - ]}, - hocon_tconf:generate(emqx_conf_schema, ConfMap) - ), - BadHeader = - "" - "\n" - "authentication = [\n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"get\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"http://127.0.0.1:8080\"\n" - "}\n" - "]\n" - "", - Conf1 = <>, + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + + OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + + OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), + Conf1 = <>, {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), - ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_headers - } - ]}, - hocon_tconf:generate(emqx_conf_schema, ConfMap1) - ), - BadHeader2 = - "" - "\n" - "authentication = \n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"get\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"http://127.0.0.1:8080\"\n" - "}\n" - "\n" - "", - Conf2 = <>, + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + + DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), + Conf2 = <>, {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_headers - } - ]}, + ?ERROR(check_http_ssl_opts), hocon_tconf:generate(emqx_conf_schema, ConfMap2) ), + + BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), + Conf3 = <>, + {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap3) + ), + + BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), + Conf4 = <>, + {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap4) + ), ok. doc_gen_test() -> @@ -164,3 +136,6 @@ doc_gen_test() -> ok end }. + +to_bin(Format, Args) -> + iolist_to_binary(io_lib:format(Format, Args)). diff --git a/apps/emqx_prometheus/TODO b/apps/emqx_prometheus/TODO deleted file mode 100644 index a868fba7e..000000000 --- a/apps/emqx_prometheus/TODO +++ /dev/null @@ -1,2 +0,0 @@ -1. Add more VM Metrics -2. Add more emqx Metrics From cea0502160ddf9cd9aed83580685eb3d34dd25b7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Apr 2023 16:23:47 +0800 Subject: [PATCH 011/194] chore: update apps/emqx_conf/README.md Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- apps/emqx_conf/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_conf/README.md b/apps/emqx_conf/README.md index 726fd72cd..f1efe7987 100644 --- a/apps/emqx_conf/README.md +++ b/apps/emqx_conf/README.md @@ -2,8 +2,7 @@ This application provides configuration management capabilities for EMQX. -This includes, during compilation: -- Read all configuration schemas and generate the following files: +At compile time it reads all configuration schemas and generates the following files: * `config-en.md`: documentation for all configuration options. * `schema-en.json`: JSON description of all configuration schema options. * `emqx.conf.example`: an example of a complete configuration file. From ffa60d0aa49db0ebbc7d0405ed82b7b7d1fb1044 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 21 Apr 2023 19:26:07 +0800 Subject: [PATCH 012/194] chore: upgrade dashboard to e1.0.6-beta.1 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7c0a133f4..a941af3b8 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.2.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6-beta.1 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6-beta.2 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From 701a1f65f994d6b152c341f235d749b2bdaa32ac Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 20 Apr 2023 13:57:47 +0200 Subject: [PATCH 013/194] chore: Hide config shared_dispatch_ack_enabled --- apps/emqx/src/emqx_schema.erl | 6 ++---- apps/emqx/src/emqx_shared_sub.erl | 4 ++-- apps/emqx/test/emqx_config_SUITE.erl | 1 - rel/i18n/emqx_schema.hocon | 6 +++--- rel/i18n/zh/emqx_schema.hocon | 5 +++-- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ace6d3332..4c314456b 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1489,10 +1489,8 @@ fields("broker") -> sc( boolean(), #{ - %% TODO: deprecated => {since, "5.1.0"} - %% in favor of session message re-dispatch at termination - %% we will stop supporting dispatch acks for shared - %% subscriptions. + deprecated => {since, "5.1.0"}, + importance => ?IMPORTANCE_HIDDEN, default => false, desc => ?DESC(broker_shared_dispatch_ack_enabled) } diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index d7dc8c5a6..997364898 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -165,7 +165,7 @@ strategy(Group) -> -spec ack_enabled() -> boolean(). ack_enabled() -> - emqx:get_config([broker, shared_dispatch_ack_enabled]). + emqx:get_config([broker, shared_dispatch_ack_enabled], false). do_dispatch(SubPid, _Group, Topic, Msg, _Type) when SubPid =:= self() -> %% Deadlock otherwise @@ -181,7 +181,7 @@ do_dispatch(SubPid, _Group, Topic, Msg, retry) -> do_dispatch(SubPid, Group, Topic, Msg, fresh) -> case ack_enabled() of true -> - %% FIXME: replace with `emqx_shared_sub_proto:dispatch_with_ack' in 5.2 + %% TODO: delete this clase after 5.1.0 do_dispatch_with_ack(SubPid, Group, Topic, Msg); false -> send(SubPid, Topic, {deliver, Topic, Msg}) diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 7befd7a16..b54f67f07 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -50,7 +50,6 @@ t_fill_default_values(_) -> }, <<"route_batch_clean">> := false, <<"session_locking_strategy">> := quorum, - <<"shared_dispatch_ack_enabled">> := false, <<"shared_subscription_strategy">> := round_robin } }, diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 9c2a1530d..76cce8e78 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1373,9 +1373,9 @@ persistent_session_store_message_gc_interval.label: """Message GC interval""" broker_shared_dispatch_ack_enabled.desc: -"""Deprecated, will be removed in 5.1. -Enable/disable shared dispatch acknowledgement for QoS 1 and QoS 2 messages. -This should allow messages to be dispatched to a different subscriber in the group in case the picked (based on `shared_subscription_strategy`) subscriber is offline.""" +"""Deprecated. +This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected. +However it's no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group.""" base_listener_enable_authn.desc: """Set true (default) to enable client authentication on this listener, the authentication diff --git a/rel/i18n/zh/emqx_schema.hocon b/rel/i18n/zh/emqx_schema.hocon index 3616abe91..1e42a4589 100644 --- a/rel/i18n/zh/emqx_schema.hocon +++ b/rel/i18n/zh/emqx_schema.hocon @@ -1313,9 +1313,10 @@ persistent_session_store_message_gc_interval.label: """消息清理间隔""" broker_shared_dispatch_ack_enabled.desc: -"""该配置项已废弃，会在 5.1 中移除。 +"""该配置项已废弃。 启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 -开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" +该配置最初设计用于避免将消息派发给客户端离线状态下的会话中去。 +但新版本中，已做增强：在一个会话结束时，会话中的消息会重新派发到组内的其他会话中 -- 使这个老配置失去存在的意义。""" base_listener_enable_authn.desc: """配置 true （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。 From df31a8a3424b38794fae620a6db459afb72651b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Fri, 21 Apr 2023 20:29:14 +0800 Subject: [PATCH 014/194] test: conf_schema_tests failed --- apps/emqx_conf/test/emqx_conf_schema_tests.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 667d1766f..192aa0e93 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -25,8 +25,8 @@ array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( - fun(Nodes) -> - ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), + fun({Seeds, Nodes}) -> + ConfFile = to_bin(?BASE_CONF, [Seeds, Nodes]), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), ClusterDiscovery = proplists:get_value( @@ -43,7 +43,7 @@ array_nodes_test() -> Nodes ) end, - [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] + [{["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"}] ), ok. @@ -79,7 +79,7 @@ array_nodes_test() -> ). authn_validations_test() -> - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, [["emqx1@127.0.0.1"], "emqx1@127.0.0.1,emqx1@127.0.0.1"]), OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, From ceafc52ad6b3c8ef72abdbe92e960a4c8c1bbd6d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 19:59:23 +0200 Subject: [PATCH 015/194] refactor: use emqx_utils_ets for ets table creation --- apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl | 2 +- scripts/merge-config.escript | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl index 9d8d1905d..b503fed88 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl @@ -36,7 +36,7 @@ init() -> OtherLangDesc0 = filelib:wildcard("desc.*.hocon", WwwStaticDir), OtherLangDesc = lists:map(fun(F) -> filename:join([WwwStaticDir, F]) end, OtherLangDesc0), Files = [EngDesc | OtherLangDesc], - ?MODULE = ets:new(?MODULE, [named_table, public, set, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]), ok = lists:foreach(fun(F) -> load_desc(?MODULE, F) end, Files). %% @doc Load the description of the configuration items from the file. diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index b3c214dd7..14ec979f2 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -110,14 +110,14 @@ merge_desc_files_per_lang(Lang) -> BaseConf = <<"">>, Cfgs0 = get_all_desc_files(Lang), Conf = do_merge_desc_files_per_lang(BaseConf, Cfgs0), - OutputFile = case Lang of + OutputFile = case Lang of "en" -> %% en desc will always be in the priv dir of emqx_dashboard "apps/emqx_dashboard/priv/desc.en.hocon"; "zh" -> %% so far we inject zh desc as if it's extracted from dashboard package %% TODO: remove this when we have zh translation moved to dashboard package - "apps/emqx_dashboard/priv/www/static/desc.zh.hocon" + "apps/emqx_dashboard/priv/www/static/desc.zh.hocon" end, ok = filelib:ensure_dir(OutputFile), ok = file:write_file(OutputFile, Conf). From a6d72b178bb32bd90fdf8931b168db74afbebba3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 20:04:22 +0200 Subject: [PATCH 016/194] chore: delete old script split-i19n-files.escript is no longer needed --- scripts/split-i18n-files.escript | 84 -------------------------------- 1 file changed, 84 deletions(-) delete mode 100755 scripts/split-i18n-files.escript diff --git a/scripts/split-i18n-files.escript b/scripts/split-i18n-files.escript deleted file mode 100755 index b9f558925..000000000 --- a/scripts/split-i18n-files.escript +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env escript - -%% This script is for one-time use. -%% will be deleted after the migration is done. - --mode(compile). - -main([]) -> - %% we need to parse hocon - %% so we'll just add all compiled libs to path - code:add_pathsz(find_ebin_paths("_build/default/lib/*")), - Files = filelib:wildcard("rel/i18n/*.hocon"), - ok = lists:foreach(fun split_file/1, Files), - ok. - -find_ebin_paths(DirPattern) -> - LibDirs = filelib:wildcard(DirPattern), - lists:filtermap(fun add_ebin/1, LibDirs). - -add_ebin(Dir) -> - EbinDir = filename:join(Dir, "ebin"), - case filelib:is_dir(EbinDir) of - true -> {true, EbinDir}; - false -> false - end. - -split_file(Path) -> - {ok, DescMap} = hocon:load(Path), - [{Module, Descs}] = maps:to_list(DescMap), - try - ok = split(Path, Module, <<"en">>, Descs), - ok = split(Path, Module, <<"zh">>, Descs) - catch - throw : already_done -> - ok - end. - -split(Path, Module, Lang, Fields) when is_map(Fields) -> - split(Path, Module, Lang, maps:to_list(Fields)); -split(Path, Module, Lang, Fields) when is_list(Fields) -> - Split = lists:map(fun({Name, Desc})-> do_split(Path, Name, Lang, Desc) end, Fields), - IoData = [Module, " {\n\n", Split, "}\n"], - %% assert it's a valid HOCON object - {ok, _} = hocon:binary(IoData), - %io:format(user, "~s", [IoData]). - WritePath = case Lang of - <<"en">> -> - Path; - <<"zh">> -> - rename(Path, "zh") - end, - ok = filelib:ensure_dir(WritePath), - ok = file:write_file(WritePath, IoData), - ok. - -rename(FilePath, Lang) -> - Dir = filename:dirname(FilePath), - BaseName = filename:basename(FilePath), - filename:join([Dir, Lang, BaseName]). - -do_split(_Path, _Name, _Lang, #{<<"desc">> := Desc}) when is_binary(Desc) -> - throw(already_done); -do_split(Path, Name, Lang, #{<<"desc">> := Desc} = D) -> - try - Label = maps:get(<<"label">>, D, #{}), - DescL = maps:get(Lang, Desc), - LabelL = maps:get(Lang, Label, undefined), - [fmt([Name, ".desc:\n"], DescL), - fmt([Name, ".label:\n"], LabelL) - ] - catch - C : E : S-> - erlang:raise(C, {Path, Name, E}, S) - end. - - -tq() -> - "\"\"\"". - -fmt(_Key, undefined) -> - []; -fmt(Key, Content) -> - [Key, tq(), Content, tq(), "\n\n"]. - From 81340edbca43f042ba76b9de1f4bf9d45a0ba05e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 22 Apr 2023 09:08:07 +0200 Subject: [PATCH 017/194] docs: add changelog --- changes/ce/fix-10462.en.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ce/fix-10462.en.md diff --git a/changes/ce/fix-10462.en.md b/changes/ce/fix-10462.en.md new file mode 100644 index 000000000..9e7922be2 --- /dev/null +++ b/changes/ce/fix-10462.en.md @@ -0,0 +1,4 @@ +Deprecate config `broker.shared_dispatch_ack_enabled`. +This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected. +However since v5.0.9, this feature is no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group. +See also: https://github.com/emqx/emqx/pull/9104 From 6beb9e00b6fe83a82a477e5eaef50ce789bb0724 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sat, 22 Apr 2023 20:09:40 +0200 Subject: [PATCH 018/194] chore: e5.0.3-alpha.2 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ea79dcd0e..d1f1a93ae 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3-alpha.1"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.2"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 5074825075363047dacba56f11bd46ed166d1f31 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 23 Apr 2023 09:56:24 +0800 Subject: [PATCH 019/194] feat(opents): OpenTSDB integration --- apps/emqx_bridge_opents/.gitignore | 19 ++ apps/emqx_bridge_opents/BSL.txt | 94 +++++++++ apps/emqx_bridge_opents/README.md | 9 + .../etc/emqx_bridge_opents.conf | 0 apps/emqx_bridge_opents/rebar.config | 8 + .../src/emqx_bridge_opents.app.src | 15 ++ .../src/emqx_bridge_opents.erl | 85 ++++++++ .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 3 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 17 +- .../src/emqx_ee_connector_opents.erl | 182 ++++++++++++++++++ rel/i18n/emqx_bridge_opents.hocon | 26 +++ rel/i18n/emqx_ee_connector_opents.hocon | 20 ++ rel/i18n/zh/emqx_bridge_opents.hocon | 26 +++ rel/i18n/zh/emqx_ee_connector_opents.hocon | 20 ++ 14 files changed, 520 insertions(+), 4 deletions(-) create mode 100644 apps/emqx_bridge_opents/.gitignore create mode 100644 apps/emqx_bridge_opents/BSL.txt create mode 100644 apps/emqx_bridge_opents/README.md create mode 100644 apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf create mode 100644 apps/emqx_bridge_opents/rebar.config create mode 100644 apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src create mode 100644 apps/emqx_bridge_opents/src/emqx_bridge_opents.erl create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl create mode 100644 rel/i18n/emqx_bridge_opents.hocon create mode 100644 rel/i18n/emqx_ee_connector_opents.hocon create mode 100644 rel/i18n/zh/emqx_bridge_opents.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_opents.hocon diff --git a/apps/emqx_bridge_opents/.gitignore b/apps/emqx_bridge_opents/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_bridge_opents/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_bridge_opents/BSL.txt b/apps/emqx_bridge_opents/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_opents/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_opents/README.md b/apps/emqx_bridge_opents/README.md new file mode 100644 index 000000000..a172cba15 --- /dev/null +++ b/apps/emqx_bridge_opents/README.md @@ -0,0 +1,9 @@ +emqx_bridge_opentsdb +===== + +An OTP application + +Build +----- + + $ rebar3 compile diff --git a/apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf b/apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_opents/rebar.config b/apps/emqx_bridge_opents/rebar.config new file mode 100644 index 000000000..d7bd4560f --- /dev/null +++ b/apps/emqx_bridge_opents/rebar.config @@ -0,0 +1,8 @@ +{erl_opts, [debug_info]}. + +{deps, [ + {opentsdb, {git, "https://github.com/emqx/opentsdb-client-erl", {tag, "v0.5.1"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src new file mode 100644 index 000000000..d001446b3 --- /dev/null +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -0,0 +1,15 @@ +{application, emqx_bridge_opents, [ + {description, "EMQX Enterprise OpenTSDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + opentsdb + ]}, + {env, []}, + {modules, []}, + + {licenses, ["BSL"]}, + {links, []} +]}. diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl new file mode 100644 index 000000000..9001e391c --- /dev/null +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl @@ -0,0 +1,85 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_opents). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api +conn_bridge_examples(Method) -> + [ + #{ + <<"opents">> => #{ + summary => <<"OpenTSDB Bridge">>, + value => values(Method) + } + } + ]. + +values(_Method) -> + #{ + enable => true, + type => opents, + name => <<"foo">>, + server => <<"http://127.0.0.1:4242">>, + pool_size => 8, + resource_opts => #{ + worker_pool_size => 1, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES + } + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_opents". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})} + ] ++ emqx_resource_schema:fields("resource_opts") ++ + emqx_ee_connector_opents:fields(config); +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for OpenTSDB using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- +%% internal + +type_field() -> + {type, mk(enum([opents]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 440889d02..7dc8882b3 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -8,7 +8,8 @@ emqx_ee_connector, telemetry, emqx_bridge_kafka, - emqx_bridge_gcp_pubsub + emqx_bridge_gcp_pubsub, + emqx_bridge_opents ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 7fdfbba99..636166d90 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -35,7 +35,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_sqlserver, Method) + ref(emqx_ee_bridge_sqlserver, Method), + ref(emqx_bridge_opents, Method) ]. schema_modules() -> @@ -55,7 +56,8 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_sqlserver + emqx_ee_bridge_sqlserver, + emqx_bridge_opents ]. examples(Method) -> @@ -94,7 +96,8 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(sqlserver) -> emqx_ee_connector_sqlserver. +resource_type(sqlserver) -> emqx_ee_connector_sqlserver; +resource_type(opents) -> emqx_ee_connector_opents. fields(bridges) -> [ @@ -153,6 +156,14 @@ fields(bridges) -> desc => <<"Cassandra Bridge Config">>, required => false } + )}, + {opents, + mk( + hoconsc:map(name, ref(emqx_bridge_opents, "config")), + #{ + desc => <<"OpenTSDB Bridge Config">>, + required => false + } )} ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs(). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl new file mode 100644 index 000000000..457fde0a0 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl @@ -0,0 +1,182 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_opents). + +-behaviour(emqx_resource). + +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([roots/0, fields/1]). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_get_status/2 +]). + +-export([connect/1]). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, mk(binary(), #{required => true, desc => ?DESC("server")})}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {summary, mk(boolean(), #{default => true, desc => ?DESC("summary")})}, + {details, mk(boolean(), #{default => false, desc => ?DESC("details")})}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} + ]. + +%%======================================================================================== +%% `emqx_resource' API +%%======================================================================================== + +callback_mode() -> always_sync. + +is_buffer_supported() -> false. + +on_start( + InstanceId, + #{ + server := Server, + pool_size := PoolSize, + summary := Summary, + details := Details, + resource_opts := #{batch_size := BatchSize} + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_opents_connector", + connector => InstanceId, + config => emqx_utils:redact(Config) + }), + + Options = [ + {server, to_str(Server)}, + {summary, Summary}, + {details, Details}, + {max_batch_size, BatchSize}, + {pool_size, PoolSize} + ], + + State = #{poolname => InstanceId, server => Server}, + case opentsdb_connectivity(Server) of + ok -> + case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + ok -> + {ok, State}; + Error -> + Error + end; + {error, Reason} = Error -> + ?SLOG(error, #{msg => "Initiate resource failed", reason => Reason}), + Error + end. + +on_stop(InstanceId, #{poolname := PoolName} = _State) -> + ?SLOG(info, #{ + msg => "stopping_opents_connector", + connector => InstanceId + }), + emqx_plugin_libs_pool:stop_pool(PoolName). + +on_query(InstanceId, Request, State) -> + on_batch_query(InstanceId, [Request], State). + +on_batch_query( + InstanceId, + BatchReq, + State +) -> + Datas = [format_opentsdb_msg(Msg) || {_Key, Msg} <- BatchReq], + do_query(InstanceId, Datas, State). + +on_get_status(_InstanceId, #{server := Server}) -> + case opentsdb_connectivity(Server) of + ok -> + connected; + {error, Reason} -> + ?SLOG(error, #{msg => "OpenTSDB lost connection", reason => Reason}), + connecting + end. + +%%======================================================================================== +%% Helper fns +%%======================================================================================== + +do_query(InstanceId, Query, #{poolname := PoolName} = State) -> + ?TRACE( + "QUERY", + "opents_connector_received", + #{connector => InstanceId, query => Query, state => State} + ), + Result = ecpool:pick_and_do(PoolName, {opentsdb, put, [Query]}, no_handover), + + case Result of + {error, Reason} -> + ?tp( + opents_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "opents_connector_do_query_failed", + connector => InstanceId, + query => Query, + reason => Reason + }), + Result; + _ -> + ?tp( + opents_connector_query_return, + #{result => Result} + ), + Result + end. + +connect(Opts) -> + opentsdb:start_link(Opts). + +to_str(List) when is_list(List) -> + List; +to_str(Bin) when is_binary(Bin) -> + erlang:binary_to_list(Bin). + +opentsdb_connectivity(Server) -> + SvrUrl = + case Server of + <<"http://", _/binary>> -> Server; + <<"https://", _/binary>> -> Server; + _ -> "http://" ++ Server + end, + emqx_plugin_libs_rule:http_connectivity(SvrUrl). + +format_opentsdb_msg(Msg) -> + maps:with( + [ + timestamp, + metric, + tags, + value, + <<"timestamp">>, + <<"metric">>, + <<"tags">>, + <<"value">> + ], + Msg + ). diff --git a/rel/i18n/emqx_bridge_opents.hocon b/rel/i18n/emqx_bridge_opents.hocon new file mode 100644 index 000000000..ff44a9e18 --- /dev/null +++ b/rel/i18n/emqx_bridge_opents.hocon @@ -0,0 +1,26 @@ +emqx_bridge_opents { + + config_enable.desc: + """Enable or disable this bridge""" + + config_enable.label: + "Enable Or Disable Bridge" + + desc_config.desc: + """Configuration for an OpenTSDB bridge.""" + + desc_config.label: + "OpenTSDB Bridge Configuration" + + desc_type.desc: + """The Bridge Type""" + + desc_type.label: + "Bridge Type" + + desc_name.desc: + """Bridge name.""" + + desc_name.label: + "Bridge Name" +} diff --git a/rel/i18n/emqx_ee_connector_opents.hocon b/rel/i18n/emqx_ee_connector_opents.hocon new file mode 100644 index 000000000..4e51454c9 --- /dev/null +++ b/rel/i18n/emqx_ee_connector_opents.hocon @@ -0,0 +1,20 @@ +emqx_ee_connector_opents { + + server.desc: + """The URL of OpenTSDB endpoint.""" + + server.label: + "URL" + + summary.desc: + """Whether or not to return summary information.""" + + summary.label: + "Summary" + + details.desc: + """Whether or not to return detailed information.""" + + details.label: + "Details" +} diff --git a/rel/i18n/zh/emqx_bridge_opents.hocon b/rel/i18n/zh/emqx_bridge_opents.hocon new file mode 100644 index 000000000..137e687df --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_opents.hocon @@ -0,0 +1,26 @@ +emqx_bridge_opents { + + config_enable.desc: + """启用/禁用桥接""" + + config_enable.label: + "启用/禁用桥接" + + desc_config.desc: + """OpenTSDB 桥接配置""" + + desc_config.label: + "OpenTSDB 桥接配置" + + desc_type.desc: + """Bridge 类型""" + + desc_type.label: + "桥接类型" + + desc_name.desc: + """桥接名字""" + + desc_name.label: + "桥接名字" +} diff --git a/rel/i18n/zh/emqx_ee_connector_opents.hocon b/rel/i18n/zh/emqx_ee_connector_opents.hocon new file mode 100644 index 000000000..7e58da9bd --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_opents.hocon @@ -0,0 +1,20 @@ +emqx_ee_connector_opents { + + server.desc: + """服务器的地址。""" + + server.label: + "服务器地址" + + summary.desc: + """是否返回摘要信息。""" + + summary.label: + "摘要信息" + + details.desc: + """是否返回详细信息。""" + + details.label: + "详细信息" +} From d826b0921dba154007683940577a4a35ea00a350 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Apr 2023 18:16:40 +0800 Subject: [PATCH 020/194] fix(dynamo): separate the implementation of connector and client of Dynamo bridge --- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../src/emqx_ee_connector_dynamo.erl | 129 ++----------- .../src/emqx_ee_connector_dynamo_client.erl | 180 ++++++++++++++++++ 3 files changed, 196 insertions(+), 115 deletions(-) create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 9cf7eb8f4..3b07acbe0 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -251,7 +251,7 @@ directly_setup_dynamo() -> directly_query(Query) -> directly_setup_dynamo(), - emqx_ee_connector_dynamo:execute(Query, ?TABLE_BIN). + emqx_ee_connector_dynamo_client:execute(Query, ?TABLE_BIN). directly_get_payload(Key) -> case directly_query({get_item, {<<"id">>, Key}}) of diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 9a149b6f7..2170827d6 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -28,10 +28,7 @@ ]). -export([ - connect/1, - do_get_status/1, - do_async_reply/2, - worker_do_query/4 + connect/1 ]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -40,10 +37,6 @@ default_port => 8000 }). --ifdef(TEST). --export([execute/2]). --endif. - %%===================================================================== %% Hocon schema roots() -> @@ -126,45 +119,39 @@ on_stop(InstanceId, #{poolname := PoolName} = _State) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstanceId, Query, State) -> - do_query(InstanceId, Query, handover, State). + do_query(InstanceId, Query, sync, State). -on_query_async(InstanceId, Query, Reply, State) -> +on_query_async(InstanceId, Query, ReplyCtx, State) -> do_query( InstanceId, Query, - {handover_async, {?MODULE, do_async_reply, [Reply]}}, + {async, ReplyCtx}, State ). %% we only support batch insert on_batch_query(InstanceId, [{send_message, _} | _] = Query, State) -> - do_query(InstanceId, Query, handover, State); + do_query(InstanceId, Query, sync, State); on_batch_query(_InstanceId, Query, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. %% we only support batch insert -on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, Reply, State) -> +on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, ReplyCtx, State) -> do_query( InstanceId, Query, - {handover_async, {?MODULE, do_async_reply, [Reply]}}, + {async, ReplyCtx}, State ); on_batch_query_async(_InstanceId, Query, _Reply, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. on_get_status(_InstanceId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), + Health = emqx_plugin_libs_pool:health_check_ecpool_workers( + Pool, {emqx_ee_connector_dynamo_client, is_connected, []} + ), status_result(Health). -do_get_status(_Conn) -> - %% because the dynamodb driver connection process is the ecpool worker self - %% so we must call the checker function inside the worker - case erlcloud_ddb2:list_tables() of - {ok, _} -> true; - _ -> false - end. - status_result(_Status = true) -> connected; status_result(_Status = false) -> connecting. @@ -185,8 +172,8 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {?MODULE, worker_do_query, [Table, Query, Templates]}, - ApplyMode + {emqx_ee_connector_dynamo_client, query, [ApplyMode, Table, Query, Templates]}, + no_handover ), case Result of @@ -210,47 +197,10 @@ do_query( Result end. -worker_do_query(_Client, Table, Query0, Templates) -> - try - Query = apply_template(Query0, Templates), - execute(Query, Table) - catch - _Type:Reason -> - {error, {unrecoverable_error, {invalid_request, Reason}}} - end. - -%% some simple query commands for authn/authz or test -execute({insert_item, Msg}, Table) -> - Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Table, Item); -execute({delete_item, Key}, Table) -> - erlcloud_ddb2:delete_item(Table, Key); -execute({get_item, Key}, Table) -> - erlcloud_ddb2:get_item(Table, Key); -%% commands for data bridge query or batch query -execute({send_message, Msg}, Table) -> - Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Table, Item); -execute([{put, _} | _] = Msgs, Table) -> - %% type of batch_write_item argument :: batch_write_item_request_items() - %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item()) - %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())} - %% batch_write_item_request() :: {put, item()} | {delete, key()} - erlcloud_ddb2:batch_write_item({Table, Msgs}). - connect(Opts) -> - #{ - aws_access_key_id := AccessKeyID, - aws_secret_access_key := SecretAccessKey, - host := Host, - port := Port, - schema := Schema - } = proplists:get_value(config, Opts), - erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), - - %% The dynamodb driver uses caller process as its connection process - %% so at here, the connection process is the ecpool worker self - {ok, self()}. + Options = proplists:get_value(config, Opts), + {ok, _Pid} = Result = emqx_ee_connector_dynamo_client:start_link(Options), + Result. parse_template(Config) -> Templates = @@ -283,54 +233,5 @@ get_host_schema("https://" ++ Server) -> get_host_schema(Server) -> {"http://", Server}. -apply_template({Key, Msg} = Req, Templates) -> - case maps:get(Key, Templates, undefined) of - undefined -> - Req; - Template -> - {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} - end; -%% now there is no batch delete, so -%% 1. we can simply replace the `send_message` to `put` -%% 2. convert the message to in_item() here, not at the time when calling `batch_write_items`, -%% so we can reduce some list map cost -apply_template([{send_message, _Msg} | _] = Msgs, Templates) -> - lists:map( - fun(Req) -> - {_, Msg} = apply_template(Req, Templates), - {put, convert_to_item(Msg)} - end, - Msgs - ). - -convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> - maps:fold( - fun - (_K, <<>>, AccIn) -> - AccIn; - (K, V, AccIn) -> - [{convert2binary(K), convert2binary(V)} | AccIn] - end, - [], - Msg - ); -convert_to_item(MsgBin) when is_binary(MsgBin) -> - Msg = emqx_utils_json:decode(MsgBin), - convert_to_item(Msg); -convert_to_item(Item) -> - erlang:throw({invalid_item, Item}). - -convert2binary(Value) when is_atom(Value) -> - erlang:atom_to_binary(Value, utf8); -convert2binary(Value) when is_binary(Value); is_number(Value) -> - Value; -convert2binary(Value) when is_list(Value) -> - unicode:characters_to_binary(Value); -convert2binary(Value) when is_map(Value) -> - emqx_utils_json:encode(Value). - -do_async_reply(Result, {ReplyFun, [Context]}) -> - ReplyFun(Context, Result). - redact(Data) -> emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl new file mode 100644 index 000000000..e0d8ee4bf --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl @@ -0,0 +1,180 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_connector_dynamo_client). + +-behaviour(gen_server). + +%% API +-export([ + start_link/1, + is_connected/1, + query/5, + query/4 +]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + format_status/2 +]). + +-ifdef(TEST). +-export([execute/2]). +-endif. + +%%%=================================================================== +%%% API +%%%=================================================================== +is_connected(Pid) -> + try + gen_server:call(Pid, is_connected) + catch + _:_ -> + false + end. + +query(Pid, sync, Table, Query, Templates) -> + query(Pid, Table, Query, Templates); +query(Pid, {async, ReplyCtx}, Table, Query, Templates) -> + gen_server:cast(Pid, {query, Table, Query, Templates, ReplyCtx}). + +query(Pid, Table, Query, Templates) -> + gen_server:call(Pid, {query, Table, Query, Templates}, infinity). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts Bridge which transfer data to DynamoDB +%% @endn +%%-------------------------------------------------------------------- +start_link(Options) -> + gen_server:start_link(?MODULE, Options, []). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%% Initialize dynamodb data bridge +init(#{ + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, + host := Host, + port := Port, + schema := Schema +}) -> + erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), + {ok, #{}}. + +handle_call(is_connected, _From, State) -> + _ = erlcloud_ddb2:list_tables(), + {reply, true, State}; +handle_call({query, Table, Query, Templates}, _From, State) -> + Result = do_query(Table, Query, Templates), + {reply, Result, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast({query, Table, Query, Templates, {ReplyFun, [Context]}}, State) -> + Result = do_query(Table, Query, Templates), + ReplyFun(Context, Result), + {noreply, State}; +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +-spec format_status( + Opt :: normal | terminate, + Status :: list() +) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +do_query(Table, Query0, Templates) -> + try + Query = apply_template(Query0, Templates), + execute(Query, Table) + catch + _Type:Reason -> + {error, {unrecoverable_error, {invalid_request, Reason}}} + end. + +%% some simple query commands for authn/authz or test +execute({insert_item, Msg}, Table) -> + Item = convert_to_item(Msg), + erlcloud_ddb2:put_item(Table, Item); +execute({delete_item, Key}, Table) -> + erlcloud_ddb2:delete_item(Table, Key); +execute({get_item, Key}, Table) -> + erlcloud_ddb2:get_item(Table, Key); +%% commands for data bridge query or batch query +execute({send_message, Msg}, Table) -> + Item = convert_to_item(Msg), + erlcloud_ddb2:put_item(Table, Item); +execute([{put, _} | _] = Msgs, Table) -> + %% type of batch_write_item argument :: batch_write_item_request_items() + %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item()) + %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())} + %% batch_write_item_request() :: {put, item()} | {delete, key()} + erlcloud_ddb2:batch_write_item({Table, Msgs}). + +apply_template({Key, Msg} = Req, Templates) -> + case maps:get(Key, Templates, undefined) of + undefined -> + Req; + Template -> + {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} + end; +%% now there is no batch delete, so +%% 1. we can simply replace the `send_message` to `put` +%% 2. convert the message to in_item() here, not at the time when calling `batch_write_items`, +%% so we can reduce some list map cost +apply_template([{send_message, _Msg} | _] = Msgs, Templates) -> + lists:map( + fun(Req) -> + {_, Msg} = apply_template(Req, Templates), + {put, convert_to_item(Msg)} + end, + Msgs + ). + +convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> + maps:fold( + fun + (_K, <<>>, AccIn) -> + AccIn; + (K, V, AccIn) -> + [{convert2binary(K), convert2binary(V)} | AccIn] + end, + [], + Msg + ); +convert_to_item(MsgBin) when is_binary(MsgBin) -> + Msg = emqx_utils_json:decode(MsgBin), + convert_to_item(Msg); +convert_to_item(Item) -> + erlang:throw({invalid_item, Item}). + +convert2binary(Value) when is_atom(Value) -> + erlang:atom_to_binary(Value, utf8); +convert2binary(Value) when is_binary(Value); is_number(Value) -> + Value; +convert2binary(Value) when is_list(Value) -> + unicode:characters_to_binary(Value); +convert2binary(Value) when is_map(Value) -> + emqx_utils_json:encode(Value). From 0b46acda87716f89cff6ee1e5ec273bf6c11873e Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 23 Apr 2023 09:57:47 +0800 Subject: [PATCH 021/194] test(opents): add test cases for OpenTSDB --- .ci/docker-compose-file/.env | 1 + .../docker-compose-opents.yaml | 9 + .../docker-compose-toxiproxy.yaml | 1 + .ci/docker-compose-file/toxiproxy.json | 6 + .github/workflows/run_test_cases.yaml | 1 + apps/emqx_bridge_opents/.gitignore | 19 - apps/emqx_bridge_opents/docker-ct | 2 + .../etc/emqx_bridge_opents.conf | 0 .../test/emqx_bridge_opents_SUITE.erl | 363 ++++++++++++++++++ .../src/emqx_ee_connector_opents.erl | 16 +- mix.exs | 5 +- rebar.config.erl | 2 + ...con => emqx_bridge_opents_connector.hocon} | 2 +- scripts/ct/run.sh | 9 +- scripts/find-apps.sh | 3 + 15 files changed, 410 insertions(+), 29 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-opents.yaml delete mode 100644 apps/emqx_bridge_opents/.gitignore create mode 100644 apps/emqx_bridge_opents/docker-ct delete mode 100644 apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf create mode 100644 apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl rename rel/i18n/{emqx_ee_connector_opents.hocon => emqx_bridge_opents_connector.hocon} (91%) diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index d33637ea0..3b00b454f 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -7,6 +7,7 @@ INFLUXDB_TAG=2.5.0 TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 +OPENTS_TAG=9aa7f88 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server SQLSERVER_TAG=2019-CU19-ubuntu-20.04 diff --git a/.ci/docker-compose-file/docker-compose-opents.yaml b/.ci/docker-compose-file/docker-compose-opents.yaml new file mode 100644 index 000000000..545aeb015 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-opents.yaml @@ -0,0 +1,9 @@ +version: '3.9' + +services: + opents_server: + container_name: opents + image: petergrace/opentsdb-docker:${OPENTS_TAG} + restart: always + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index ba5e831a5..a1ae41e2c 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -26,6 +26,7 @@ services: - 19876:9876 - 19042:9042 - 19142:9142 + - 14242:4242 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index da2dff763..f6b31da4c 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -101,5 +101,11 @@ "listen": "0.0.0.0:1433", "upstream": "sqlserver:1433", "enabled": true + }, + { + "name": "opents", + "listen": "0.0.0.0:4242", + "upstream": "opents:4242", + "enabled": true } ] diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index fb4f264e7..f7b775f08 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -168,6 +168,7 @@ jobs: REDIS_TAG: "7.0" INFLUXDB_TAG: "2.5.0" TDENGINE_TAG: "3.0.2.4" + OPENTS_TAG: "9aa7f88" PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} diff --git a/apps/emqx_bridge_opents/.gitignore b/apps/emqx_bridge_opents/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/apps/emqx_bridge_opents/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/apps/emqx_bridge_opents/docker-ct b/apps/emqx_bridge_opents/docker-ct new file mode 100644 index 000000000..fc68b978e --- /dev/null +++ b/apps/emqx_bridge_opents/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +opents diff --git a/apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf b/apps/emqx_bridge_opents/etc/emqx_bridge_opents.conf deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl new file mode 100644 index 000000000..6f444b93e --- /dev/null +++ b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl @@ -0,0 +1,363 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_opents_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% DB defaults +-define(BATCH_SIZE, 10). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, with_batch}, + {group, without_batch} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {with_batch, TCs}, + {without_batch, TCs} + ]. + +init_per_group(with_batch, Config0) -> + Config = [{batch_size, ?BATCH_SIZE} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{batch_size, 1} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + delete_bridge(Config), + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("OPENTS_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("OPENTS_PORT", "4242")), + + Config0 = [ + {opents_host, Host}, + {opents_port, Port}, + {proxy_name, "opents"} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"opents">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + {Name, OpenTSConf} = opents_config(BridgeType, Config0), + Config = + [ + {opents_config, OpenTSConf}, + {opents_bridge_type, BridgeType}, + {opents_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_opents); + _ -> + {skip, no_opents} + end + end. + +opents_config(BridgeType, Config) -> + Port = integer_to_list(?config(opents_port, Config)), + Server = "http://" ++ ?config(opents_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = ?config(batch_size, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " resource_opts = {\n" + " request_timeout = 500ms\n" + " batch_size = ~b\n" + " query_mode = sync\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + BatchSize + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(opents_bridge_type, Config), + Name = ?config(opents_name, Config), + Config0 = ?config(opents_config, Config), + Config1 = emqx_utils_maps:deep_merge(Config0, Overrides), + emqx_bridge:create(BridgeType, Name, Config1). + +delete_bridge(Config) -> + BridgeType = ?config(opents_bridge_type, Config), + Name = ?config(opents_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(opents_name, Config), + BridgeType = ?config(opents_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + query_resource(Config, Request, 1_000). + +query_resource(Config, Request, Timeout) -> + Name = ?config(opents_name, Config), + BridgeType = ?config(opents_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => Timeout}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + SentData = make_data(), + ?check_trace( + begin + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + {ok, 200, #{failed := 0, success := 1}}, Result + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(opents_connector_query_return, Trace0), + ?assertMatch([#{result := {ok, 200, #{failed := 0, success := 1}}}], Trace), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(opents_bridge_type, Config), + Name = ?config(opents_name, Config), + OpentsConfig0 = ?config(opents_config, Config), + OpentsConfig = OpentsConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(OpentsConfig) + ), + SentData = make_data(), + ?check_trace( + begin + Request = {send_message, SentData}, + Res0 = query_resource(Config, Request, 2_500), + ?assertMatch( + {ok, 200, #{failed := 0, success := 1}}, Res0 + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(opents_connector_query_return, Trace0), + ?assertMatch([#{result := {ok, 200, #{failed := 0, success := 1}}}], Trace), + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + + Name = ?config(opents_name, Config), + BridgeType = ?config(opents_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + ok. + +t_create_disconnected(Config) -> + BridgeType = proplists:get_value(bridge_type, Config, <<"opents">>), + Config1 = lists:keyreplace(opents_port, 1, Config, {opents_port, 61234}), + {_Name, OpenTSConf} = opents_config(BridgeType, Config1), + + Config2 = lists:keyreplace(opents_config, 1, Config1, {opents_config, OpenTSConf}), + ?assertMatch({ok, _}, create_bridge(Config2)), + + Name = ?config(opents_name, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceID)), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge(Config), + SentData = make_data(), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch({error, _}, Result), + ok + end), + ok. + +t_write_timeout(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge( + Config, + #{ + <<"resource_opts">> => #{ + <<"request_timeout">> => 500, + <<"resume_interval">> => 100, + <<"health_check_interval">> => 100 + } + } + ), + SentData = make_data(), + emqx_common_test_helpers:with_failure( + timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData}) + ) + end + ), + ok. + +t_missing_data(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch( + {error, {400, #{failed := 1, success := 0}}}, + Result + ), + ok. + +t_bad_data(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Data = maps:without([metric], make_data()), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, Data), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + + ?assertMatch( + {error, {400, #{failed := 1, success := 0}}}, Result + ), + ok. + +make_data() -> + make_data(<<"cpu">>, 12). + +make_data(Metric, Value) -> + #{ + metric => Metric, + tags => #{ + <<"host">> => <<"serverA">> + }, + value => Value + }. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl index 457fde0a0..dfc960493 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl @@ -108,13 +108,15 @@ on_batch_query( do_query(InstanceId, Datas, State). on_get_status(_InstanceId, #{server := Server}) -> - case opentsdb_connectivity(Server) of - ok -> - connected; - {error, Reason} -> - ?SLOG(error, #{msg => "OpenTSDB lost connection", reason => Reason}), - connecting - end. + Result = + case opentsdb_connectivity(Server) of + ok -> + connected; + {error, Reason} -> + ?SLOG(error, #{msg => "OpenTSDB lost connection", reason => Reason}), + connecting + end, + Result. %%======================================================================================== %% Helper fns diff --git a/mix.exs b/mix.exs index c5d6df804..e2230d55d 100644 --- a/mix.exs +++ b/mix.exs @@ -157,6 +157,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_kafka, :emqx_bridge_gcp_pubsub, :emqx_bridge_cassandra, + :emqx_bridge_opents, :emqx_bridge_clickhouse, :emqx_bridge_dynamo, :emqx_bridge_hstreamdb, @@ -182,7 +183,8 @@ defmodule EMQXUmbrella.MixProject do {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, {:crc32cer, "0.1.8", override: true}, - {:supervisor3, "1.1.12", override: true} + {:supervisor3, "1.1.12", override: true}, + {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true} ] end @@ -360,6 +362,7 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_kafka: :permanent, emqx_bridge_gcp_pubsub: :permanent, emqx_bridge_cassandra: :permanent, + emqx_bridge_opents: :permanent, emqx_bridge_clickhouse: :permanent, emqx_bridge_dynamo: :permanent, emqx_bridge_hstreamdb: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 88471c39d..3c863046f 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -81,6 +81,7 @@ is_enterprise(ee) -> true. is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; +is_community_umbrella_app("apps/emqx_bridge_opents") -> false; is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; @@ -455,6 +456,7 @@ relx_apps_per_edition(ee) -> emqx_bridge_kafka, emqx_bridge_gcp_pubsub, emqx_bridge_cassandra, + emqx_bridge_opents, emqx_bridge_clickhouse, emqx_bridge_dynamo, emqx_bridge_hstreamdb, diff --git a/rel/i18n/emqx_ee_connector_opents.hocon b/rel/i18n/emqx_bridge_opents_connector.hocon similarity index 91% rename from rel/i18n/emqx_ee_connector_opents.hocon rename to rel/i18n/emqx_bridge_opents_connector.hocon index 4e51454c9..cd82809d2 100644 --- a/rel/i18n/emqx_ee_connector_opents.hocon +++ b/rel/i18n/emqx_bridge_opents_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_opents { +emqx_bridge_opents_connector { server.desc: """The URL of OpenTSDB endpoint.""" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index ab7fff444..c1a01a593 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -115,7 +115,11 @@ case "${WHICH_APP}" in export PROFILE='emqx' fi ;; - *) + apps/emqx_bridge_opents) + ## ensure enterprise profile when testing ee applications + export PROFILE='emqx-enterprise' + ;; + *) export PROFILE="${PROFILE:-emqx}" ;; esac @@ -188,6 +192,9 @@ for dep in ${CT_DEPS}; do ODBC_REQUEST='yes' FILES+=( '.ci/docker-compose-file/docker-compose-sqlserver.yaml' ) ;; + opents) + FILES+=( '.ci/docker-compose-file/docker-compose-opents.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index bfb6ba2cc..64d28529f 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -72,6 +72,9 @@ describe_app() { runner="docker" fi case "${app}" in + apps/emqx_bridge_opents) + profile='emqx-enterprise' + ;; apps/*) if [[ -f "${app}/BSL.txt" ]]; then profile='emqx-enterprise' From 540518eac308608a310f89782c4e1ca83852af98 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 10:24:39 +0800 Subject: [PATCH 022/194] chore: add README for OpenTSDB bridge --- apps/emqx_bridge_opents/README.md | 39 ++++++++++++++++++++++++++----- scripts/ct/run.sh | 6 +---- scripts/find-apps.sh | 3 --- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge_opents/README.md b/apps/emqx_bridge_opents/README.md index a172cba15..a1d6511ee 100644 --- a/apps/emqx_bridge_opents/README.md +++ b/apps/emqx_bridge_opents/README.md @@ -1,9 +1,36 @@ -emqx_bridge_opentsdb -===== +# EMQX OpenTSDB Bridge -An OTP application +[OpenTSDB](http://opentsdb.net) is a distributed, scalable Time Series Database (TSDB) written on top of HBase. -Build ------ +OpenTSDB was written to address a common need: store, index and serve metrics collected from computer systems (network gear, operating systems, applications) at a large scale, and make this data easily accessible and graphable. - $ rebar3 compile +OpenTSDB allows you to collect thousands of metrics from tens of thousands of hosts and applications, at a high rate (every few seconds). + +OpenTSDB will never delete or downsample data and can easily store hundreds of billions of data points. + +The application is used to connect EMQX and OpenTSDB. User can create a rule and easily ingest IoT data into OpenTSDB by leveraging the +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documentation + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index c1a01a593..c153669f4 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -115,11 +115,7 @@ case "${WHICH_APP}" in export PROFILE='emqx' fi ;; - apps/emqx_bridge_opents) - ## ensure enterprise profile when testing ee applications - export PROFILE='emqx-enterprise' - ;; - *) + *) export PROFILE="${PROFILE:-emqx}" ;; esac diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 64d28529f..bfb6ba2cc 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -72,9 +72,6 @@ describe_app() { runner="docker" fi case "${app}" in - apps/emqx_bridge_opents) - profile='emqx-enterprise' - ;; apps/*) if [[ -f "${app}/BSL.txt" ]]; then profile='emqx-enterprise' From 6631fb7457efd61697656d4549886ca9fc4d4287 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 10:41:30 +0800 Subject: [PATCH 023/194] chore: update changes --- changes/ee/feat-10425.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-10425.en.md diff --git a/changes/ee/feat-10425.en.md b/changes/ee/feat-10425.en.md new file mode 100644 index 000000000..7144241df --- /dev/null +++ b/changes/ee/feat-10425.en.md @@ -0,0 +1 @@ +Implement OpenTSDB data bridge. From 932a327952d7fbaeeaecc89a10bc10440940187b Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 20 Apr 2023 15:11:37 +0800 Subject: [PATCH 024/194] chore: make spellcheck and xref happy --- .../emqx_ee_connector/src/emqx_ee_connector_opents.erl | 10 +++++----- rel/i18n/emqx_bridge_opents_connector.hocon | 4 ++-- scripts/spellcheck/dicts/emqx.txt | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl index dfc960493..633e120bd 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl @@ -75,10 +75,10 @@ on_start( {pool_size, PoolSize} ], - State = #{poolname => InstanceId, server => Server}, + State = #{pool_name => InstanceId, server => Server}, case opentsdb_connectivity(Server) of ok -> - case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of ok -> {ok, State}; Error -> @@ -89,12 +89,12 @@ on_start( Error end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName} = _State) -> ?SLOG(info, #{ msg => "stopping_opents_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstanceId, Request, State) -> on_batch_query(InstanceId, [Request], State). @@ -122,7 +122,7 @@ on_get_status(_InstanceId, #{server := Server}) -> %% Helper fns %%======================================================================================== -do_query(InstanceId, Query, #{poolname := PoolName} = State) -> +do_query(InstanceId, Query, #{pool_name := PoolName} = State) -> ?TRACE( "QUERY", "opents_connector_received", diff --git a/rel/i18n/emqx_bridge_opents_connector.hocon b/rel/i18n/emqx_bridge_opents_connector.hocon index cd82809d2..5c39d1e0e 100644 --- a/rel/i18n/emqx_bridge_opents_connector.hocon +++ b/rel/i18n/emqx_bridge_opents_connector.hocon @@ -7,13 +7,13 @@ emqx_bridge_opents_connector { "URL" summary.desc: - """Whether or not to return summary information.""" + """Whether to return summary information.""" summary.label: "Summary" details.desc: - """Whether or not to return detailed information.""" + """Whether to return detailed information.""" details.label: "Details" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 168275e1e..a9afcf6ca 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -274,3 +274,4 @@ clickhouse FormatType RocketMQ Keyspace +OpenTSDB From 6e1d6f1991d4115990bdcb741adf8c7c50d0cb64 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sun, 23 Apr 2023 11:27:59 +0800 Subject: [PATCH 025/194] chore: hide bad links in README files --- apps/emqx_bridge_cassandra/README.md | 2 ++ apps/emqx_bridge_hstreamdb/README.md | 2 ++ apps/emqx_bridge_matrix/README.md | 2 ++ apps/emqx_bridge_timescale/README.md | 2 ++ 4 files changed, 8 insertions(+) diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md index d26bd2fbb..c5a2609a5 100644 --- a/apps/emqx_bridge_cassandra/README.md +++ b/apps/emqx_bridge_cassandra/README.md @@ -11,6 +11,7 @@ The application is used to connect EMQX and Cassandra. User can create a rule and easily ingest IoT data into Cassandra by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + # HTTP APIs diff --git a/apps/emqx_bridge_hstreamdb/README.md b/apps/emqx_bridge_hstreamdb/README.md index 3a7c6b49d..520817e82 100644 --- a/apps/emqx_bridge_hstreamdb/README.md +++ b/apps/emqx_bridge_hstreamdb/README.md @@ -9,6 +9,7 @@ The application is used to connect EMQX and HStreamDB. User can create a rule and easily ingest IoT data into HStreamDB by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + # HTTP APIs diff --git a/apps/emqx_bridge_matrix/README.md b/apps/emqx_bridge_matrix/README.md index 976120ffe..339eb0605 100644 --- a/apps/emqx_bridge_matrix/README.md +++ b/apps/emqx_bridge_matrix/README.md @@ -7,6 +7,7 @@ The application is used to connect EMQX and MatrixDB. User can create a rule and easily ingest IoT data into MatrixDB by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + # HTTP APIs diff --git a/apps/emqx_bridge_timescale/README.md b/apps/emqx_bridge_timescale/README.md index 96f70f847..071cb0fa6 100644 --- a/apps/emqx_bridge_timescale/README.md +++ b/apps/emqx_bridge_timescale/README.md @@ -9,6 +9,7 @@ The application is used to connect EMQX and TimescaleDB. User can create a rule and easily ingest IoT data into TimescaleDB by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + # HTTP APIs From 5ad5d7ee8dc60c9ed3c9812a9261e83d6b5ce63e Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 21 Apr 2023 18:32:14 +0800 Subject: [PATCH 026/194] fix(opents): adjust code structure --- apps/emqx_bridge_opents/src/emqx_bridge_opents.erl | 2 +- .../emqx_bridge_opents/src/emqx_bridge_opents_connector.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- ...onnector_opents.hocon => emqx_bridge_opents_connector.hocon} | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl => apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl (99%) rename rel/i18n/zh/{emqx_ee_connector_opents.hocon => emqx_bridge_opents_connector.hocon} (90%) diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl index 9001e391c..2eb6a554f 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.erl @@ -60,7 +60,7 @@ fields("config") -> [ {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})} ] ++ emqx_resource_schema:fields("resource_opts") ++ - emqx_ee_connector_opents:fields(config); + emqx_bridge_opents_connector:fields(config); fields("post") -> [type_field(), name_field() | fields("config")]; fields("put") -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl rename to apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 633e120bd..0366c9dc2 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_opents.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_opents). +-module(emqx_bridge_opents_connector). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 636166d90..4b83fda3f 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -97,7 +97,7 @@ resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; resource_type(sqlserver) -> emqx_ee_connector_sqlserver; -resource_type(opents) -> emqx_ee_connector_opents. +resource_type(opents) -> emqx_bridge_opents_connector. fields(bridges) -> [ diff --git a/rel/i18n/zh/emqx_ee_connector_opents.hocon b/rel/i18n/zh/emqx_bridge_opents_connector.hocon similarity index 90% rename from rel/i18n/zh/emqx_ee_connector_opents.hocon rename to rel/i18n/zh/emqx_bridge_opents_connector.hocon index 7e58da9bd..f8a80b10e 100644 --- a/rel/i18n/zh/emqx_ee_connector_opents.hocon +++ b/rel/i18n/zh/emqx_bridge_opents_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_opents { +emqx_bridge_opents_connector { server.desc: """服务器的地址。""" From 5ab08876873d40908e034be609a70afe786362a7 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 23 Apr 2023 15:17:51 +0800 Subject: [PATCH 027/194] chore: add examples for RocketMQ template --- rel/i18n/emqx_ee_bridge_rocketmq.hocon | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/emqx_ee_bridge_rocketmq.hocon index 2e33e6c07..0e5b64bb1 100644 --- a/rel/i18n/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_ee_bridge_rocketmq.hocon @@ -16,8 +16,14 @@ NOTE: if the bridge is used as a rule action, `local_topic` should be left empty template { desc { - en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" - zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" + en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ.
+ The template can be any valid string with placeholders, example:
+ - ${id}, ${username}, ${clientid}, ${timestamp}
+ - {\"id\" : ${id}, \"username\" : ${username}}""" + zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ。
+ 模板可以是任意带有占位符的合法字符串, 例如:
+ - ${id}, ${username}, ${clientid}, ${timestamp}
+ - {\"id\" : ${id}, \"username\" : ${username}}""" } label { en: "Template" From 7d2c336ab7427e6b909b66f2e9f1452c3ef3baf3 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 23 Apr 2023 15:31:08 +0800 Subject: [PATCH 028/194] fix(resource): make sure resource will not crash when stopping --- apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl index a00dcdcd2..ae30c3927 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker_sup.erl @@ -141,9 +141,5 @@ ensure_disk_queue_dir_absent(ResourceId, Index) -> ok. ensure_worker_pool_removed(ResId) -> - try - gproc_pool:delete(ResId) - catch - error:badarg -> ok - end, + gproc_pool:force_delete(ResId), ok. From 7af9c18caa391aa14d4886f7b807bec0d757ca0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 15:43:18 +0800 Subject: [PATCH 029/194] fix: copy cluster-override.conf from old version --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_app.erl | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 234690374..03cd36522 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 35a79ea6e..9d6bb35d7 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -175,7 +175,7 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, - HasDeprecatedFile = maps:get(has_deprecated_file, Info, false), + HasDeprecatedFile = has_deprecated_file(Info), ?SLOG(debug, #{ msg => "copy_cluster_conf_from_core_node_success", node => Node, @@ -227,3 +227,16 @@ sync_data_from_node(Node) -> ?SLOG(emergency, #{node => Node, msg => "sync_data_from_node_failed", reason => Error}), error(Error) end. + +has_deprecated_file(#{node := Node} = Info) -> + case maps:find(has_deprecated_file, Info) of + {ok, HasDeprecatedFile} -> + HasDeprecatedFile; + error -> + %% The old version don't have emqx_config:has_deprecated_file/0 + Timeout = 5000, + {ok, File} = rpc:call( + Node, application, get_env, [emqx, cluster_override_conf_file], Timeout + ), + rpc:call(Node, filelib, is_regular, [File], Timeout) + end. From 4e7472090ba69208be2f5b462f3835ed1bfaf9b5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 18 Apr 2023 10:37:05 +0800 Subject: [PATCH 030/194] fix: refine default sql and driver name for mssql bridge --- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl | 6 +++--- scripts/ct/run.sh | 4 ++-- .../{install-odbc-driver.sh => install-msodbc-driver.sh} | 0 3 files changed, 5 insertions(+), 5 deletions(-) rename scripts/{install-odbc-driver.sh => install-msodbc-driver.sh} (100%) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl index e216299c2..49db815a6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -22,11 +22,11 @@ ]). -define(DEFAULT_SQL, << - "insert into t_mqtt_msg(msgid, topic, qos, payload)" - "values (${id}, ${topic}, ${qos}, ${payload})" + "insert into t_mqtt_msg(msgid, topic, qos, payload) " + "values ( ${id}, ${topic}, ${qos}, ${payload} )" >>). --define(DEFAULT_DRIVER, <<"ms-sqlserver-18">>). +-define(DEFAULT_DRIVER, <<"ms-sql">>). conn_bridge_examples(Method) -> [ diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4e79476e0..a85aa36af 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -193,9 +193,9 @@ for dep in ${CT_DEPS}; do done if [ "$ODBC_REQUEST" = 'yes' ]; then - INSTALL_ODBC="./scripts/install-odbc-driver.sh" + INSTALL_ODBC="./scripts/install-msodbc-driver.sh" else - INSTALL_ODBC="echo 'Driver msodbcsql driver not requested'" + INSTALL_ODBC="echo 'msodbc driver not requested'" fi F_OPTIONS="" diff --git a/scripts/install-odbc-driver.sh b/scripts/install-msodbc-driver.sh similarity index 100% rename from scripts/install-odbc-driver.sh rename to scripts/install-msodbc-driver.sh From d505b65ba81534e6cfcf2c091fb32435270ad459 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Sun, 23 Apr 2023 15:45:58 +0800 Subject: [PATCH 031/194] fix: use default health check timeout for sqlserver --- .../src/emqx_ee_connector_sqlserver.erl | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index f11441a3b..6cbd9de4e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -43,7 +43,7 @@ -export([connect/1]). %% Internal exports used to execute code with ecpool worker --export([do_get_status/2, worker_do_insert/3, do_async_reply/2]). +-export([do_get_status/1, worker_do_insert/3, do_async_reply/2]). -import(emqx_plugin_libs_rule, [str/1]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -306,10 +306,9 @@ on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> ), do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). -on_get_status(_InstanceId, #{poolname := Pool, resource_opts := ResourceOpts} = _State) -> - RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), +on_get_status(_InstanceId, #{poolname := Pool} = _State) -> Health = emqx_plugin_libs_pool:health_check_ecpool_workers( - Pool, {?MODULE, do_get_status, [RequestTimeout]}, RequestTimeout + Pool, {?MODULE, do_get_status, []} ), status_result(Health). @@ -328,9 +327,9 @@ connect(Options) -> Opts = proplists:get_value(options, Options, []), odbc:connect(ConnectStr, Opts). --spec do_get_status(connection_reference(), time_out()) -> Result :: boolean(). -do_get_status(Conn, RequestTimeout) -> - case execute(Conn, <<"SELECT 1">>, RequestTimeout) of +-spec do_get_status(connection_reference()) -> Result :: boolean(). +do_get_status(Conn) -> + case execute(Conn, <<"SELECT 1">>) of {selected, [[]], [{1}]} -> true; _ -> false end. @@ -444,6 +443,15 @@ worker_do_insert( {error, {unrecoverable_error, {invalid_request, Reason}}} end. +-spec execute(pid(), sql()) -> + updated_tuple() + | selected_tuple() + | [updated_tuple()] + | [selected_tuple()] + | {error, common_reason()}. +execute(Conn, SQL) -> + odbc:sql_query(Conn, str(SQL)). + -spec execute(pid(), sql(), time_out()) -> updated_tuple() | selected_tuple() From 38cebf2fdc5efadda7a6c46a9bf8cb58d0bfb46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 15:53:17 +0800 Subject: [PATCH 032/194] chore: add changelog for 10484 --- changes/ce/fix-10484.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ce/fix-10484.en.md diff --git a/changes/ce/fix-10484.en.md b/changes/ce/fix-10484.en.md new file mode 100644 index 000000000..d1a501384 --- /dev/null +++ b/changes/ce/fix-10484.en.md @@ -0,0 +1,3 @@ +Fix the issue that the priority of the configuration cannot be set during rolling upgrade. +For example, when authorization is modified in v5.0.21 and then upgraded v5.0.23 through rolling upgrade, +the authorization will be restored to the default. From 5593e38ed306b941d3214b96ca067ccdacad6d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 15:43:18 +0800 Subject: [PATCH 033/194] fix: copy cluster-override.conf from old version --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_app.erl | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 234690374..03cd36522 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 35a79ea6e..9d6bb35d7 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -175,7 +175,7 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, - HasDeprecatedFile = maps:get(has_deprecated_file, Info, false), + HasDeprecatedFile = has_deprecated_file(Info), ?SLOG(debug, #{ msg => "copy_cluster_conf_from_core_node_success", node => Node, @@ -227,3 +227,16 @@ sync_data_from_node(Node) -> ?SLOG(emergency, #{node => Node, msg => "sync_data_from_node_failed", reason => Error}), error(Error) end. + +has_deprecated_file(#{node := Node} = Info) -> + case maps:find(has_deprecated_file, Info) of + {ok, HasDeprecatedFile} -> + HasDeprecatedFile; + error -> + %% The old version don't have emqx_config:has_deprecated_file/0 + Timeout = 5000, + {ok, File} = rpc:call( + Node, application, get_env, [emqx, cluster_override_conf_file], Timeout + ), + rpc:call(Node, filelib, is_regular, [File], Timeout) + end. From e0fd861863835de4e4df7cb49f8c673fd78d2aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 17:20:54 +0800 Subject: [PATCH 034/194] chore: make static_check happy --- apps/emqx_conf/src/emqx_conf_app.erl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 9d6bb35d7..f7ce797b8 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -234,9 +234,7 @@ has_deprecated_file(#{node := Node} = Info) -> HasDeprecatedFile; error -> %% The old version don't have emqx_config:has_deprecated_file/0 - Timeout = 5000, - {ok, File} = rpc:call( - Node, application, get_env, [emqx, cluster_override_conf_file], Timeout - ), - rpc:call(Node, filelib, is_regular, [File], Timeout) + DataDir = emqx_conf_proto_v2:get_config(Node, [node, data_dir]), + File = filename:join([DataDir, "configs", "cluster-override.conf"]), + rpc:call(Node, filelib, is_regular, [File], 5000) end. From f96c1630e1271fac4c34cbf08951984fd7360beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 17:30:07 +0800 Subject: [PATCH 035/194] chore: pin emqx_conf to 0.17.0 --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 03cd36522..234690374 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.18"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, From b4c16d37c74ce90bf29a4137a3f98781c761f3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 17:20:54 +0800 Subject: [PATCH 036/194] chore: make static_check happy --- apps/emqx_conf/src/emqx_conf_app.erl | 2 +- .../src/proto/emqx_conf_proto_v3.erl | 114 ++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index f7ce797b8..fd0a56853 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -236,5 +236,5 @@ has_deprecated_file(#{node := Node} = Info) -> %% The old version don't have emqx_config:has_deprecated_file/0 DataDir = emqx_conf_proto_v2:get_config(Node, [node, data_dir]), File = filename:join([DataDir, "configs", "cluster-override.conf"]), - rpc:call(Node, filelib, is_regular, [File], 5000) + emqx_conf_proto_v3:file_exist(Node, File) end. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl new file mode 100644 index 000000000..802436f98 --- /dev/null +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl @@ -0,0 +1,114 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + sync_data_from_node/1, + get_config/2, + get_config/3, + get_all/1, + + update/3, + update/4, + remove_config/2, + remove_config/3, + + reset/2, + reset/3, + + get_override_config_file/1, + file_exist/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.24". + +-spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). +sync_data_from_node(Node) -> + rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. + +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath) -> + rpc:call(Node, emqx, get_config, [KeyPath]). + +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath, Default) -> + rpc:call(Node, emqx, get_config, [KeyPath, Default]). + +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). +get_all(KeyPath) -> + rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). + +-spec update( + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +update(KeyPath, UpdateReq, Opts) -> + emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). + +-spec update( + node(), + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +update(Node, KeyPath, UpdateReq, Opts) -> + rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). + +-spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +remove_config(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). + +-spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +remove_config(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). + +-spec reset(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +reset(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). + +-spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +reset(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). + +-spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). +get_override_config_file(Nodes) -> + rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). + +-spec file_exist(node(), string()) -> emqx_rpc:badrpc() | boolean(). +file_exist(Node, File) -> + rpc:call(Node, filelib, is_regular, [File], 5000). From 6dcecfed40b55a3b9f21323aca4561d72ca9db91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 17:20:54 +0800 Subject: [PATCH 037/194] chore: make static_check happy --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_conf/src/emqx_conf_app.erl | 8 +- .../src/proto/emqx_conf_proto_v2.erl | 4 + .../src/proto/emqx_conf_proto_v3.erl | 114 ++++++++++++++++++ 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index db4765e3f..11bd4aa77 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -11,6 +11,7 @@ {emqx_cm,1}. {emqx_conf,1}. {emqx_conf,2}. +{emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 9d6bb35d7..fd0a56853 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -234,9 +234,7 @@ has_deprecated_file(#{node := Node} = Info) -> HasDeprecatedFile; error -> %% The old version don't have emqx_config:has_deprecated_file/0 - Timeout = 5000, - {ok, File} = rpc:call( - Node, application, get_env, [emqx, cluster_override_conf_file], Timeout - ), - rpc:call(Node, filelib, is_regular, [File], Timeout) + DataDir = emqx_conf_proto_v2:get_config(Node, [node, data_dir]), + File = filename:join([DataDir, "configs", "cluster-override.conf"]), + emqx_conf_proto_v3:file_exist(Node, File) end. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index 97446ee9f..3bcf532f6 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, sync_data_from_node/1, get_config/2, get_config/3, @@ -41,6 +42,9 @@ introduced_in() -> "5.0.1". +deprecated_since() -> + "5.0.23". + -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl new file mode 100644 index 000000000..802436f98 --- /dev/null +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl @@ -0,0 +1,114 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_proto_v3). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + sync_data_from_node/1, + get_config/2, + get_config/3, + get_all/1, + + update/3, + update/4, + remove_config/2, + remove_config/3, + + reset/2, + reset/3, + + get_override_config_file/1, + file_exist/2 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +introduced_in() -> + "5.0.24". + +-spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). +sync_data_from_node(Node) -> + rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. + +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath) -> + rpc:call(Node, emqx, get_config, [KeyPath]). + +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> + term() | emqx_rpc:badrpc(). +get_config(Node, KeyPath, Default) -> + rpc:call(Node, emqx, get_config, [KeyPath, Default]). + +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). +get_all(KeyPath) -> + rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). + +-spec update( + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +update(KeyPath, UpdateReq, Opts) -> + emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). + +-spec update( + node(), + update_config_key_path(), + emqx_config:update_request(), + emqx_config:update_opts() +) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +update(Node, KeyPath, UpdateReq, Opts) -> + rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). + +-spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +remove_config(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). + +-spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +remove_config(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). + +-spec reset(update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. +reset(KeyPath, Opts) -> + emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). + +-spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> + {ok, emqx_config:update_result()} + | {error, emqx_config:update_error()} + | emqx_rpc:badrpc(). +reset(Node, KeyPath, Opts) -> + rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). + +-spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). +get_override_config_file(Nodes) -> + rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). + +-spec file_exist(node(), string()) -> emqx_rpc:badrpc() | boolean(). +file_exist(Node, File) -> + rpc:call(Node, filelib, is_regular, [File], 5000). From 8bfee903223780dc5492d289880b4b5aa8f76713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 23 Apr 2023 17:20:54 +0800 Subject: [PATCH 038/194] chore: make static_check happy --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index db4765e3f..11bd4aa77 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -11,6 +11,7 @@ {emqx_cm,1}. {emqx_conf,1}. {emqx_conf,2}. +{emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index 97446ee9f..3bcf532f6 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, sync_data_from_node/1, get_config/2, get_config/3, @@ -41,6 +42,9 @@ introduced_in() -> "5.0.1". +deprecated_since() -> + "5.0.23". + -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). From c2e35a42b0102db0c0053289b3129b7e05e065d8 Mon Sep 17 00:00:00 2001 From: firest Date: Sun, 23 Apr 2023 17:47:00 +0800 Subject: [PATCH 039/194] fix(limiter): optimize the instance of limiter We can reduce a limiter container with all types are `infinity` to just a `infinity` atom --- apps/emqx/src/emqx_channel.erl | 4 +- apps/emqx/src/emqx_connection.erl | 86 ++++++++++--------- .../src/emqx_limiter_container.erl | 45 +++++++--- apps/emqx/src/emqx_ws_connection.erl | 85 +++++++++--------- apps/emqx/test/emqx_connection_SUITE.erl | 30 ++++--- apps/emqx/test/emqx_ws_connection_SUITE.erl | 7 +- 6 files changed, 153 insertions(+), 104 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 8a936067e..862b72c06 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -89,7 +89,7 @@ %% Authentication Data Cache auth_cache :: maybe(map()), %% Quota checkers - quota :: maybe(emqx_limiter_container:limiter()), + quota :: emqx_limiter_container:limiter(), %% Timers timers :: #{atom() => disabled | maybe(reference())}, %% Conn State @@ -760,7 +760,7 @@ do_publish( handle_out(disconnect, RC, Channel) end. -ensure_quota(_, Channel = #channel{quota = undefined}) -> +ensure_quota(_, Channel = #channel{quota = infinity}) -> Channel; ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> Cnt = lists:foldl( diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 27b6f3e84..79654e510 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -111,7 +111,7 @@ listener :: {Type :: atom(), Name :: atom()}, %% Limiter - limiter :: maybe(limiter()), + limiter :: limiter(), %% limiter buffer for overload use limiter_buffer :: queue:queue(pending_req()), @@ -974,55 +974,61 @@ handle_cast(Req, State) -> list(any()), state() ) -> _. + +check_limiter( + _Needs, + Data, + WhenOk, + Msgs, + #state{limiter = infinity} = State +) -> + WhenOk(Data, Msgs, State); check_limiter( Needs, Data, WhenOk, Msgs, - #state{ - limiter = Limiter, - limiter_timer = LimiterTimer, - limiter_buffer = Cache - } = State -) when Limiter =/= undefined -> - case LimiterTimer of - undefined -> - case emqx_limiter_container:check_list(Needs, Limiter) of - {ok, Limiter2} -> - WhenOk(Data, Msgs, State#state{limiter = Limiter2}); - {pause, Time, Limiter2} -> - ?SLOG(debug, #{ - msg => "pause_time_dueto_rate_limit", - needs => Needs, - time_in_ms => Time - }), + #state{limiter_timer = undefined, limiter = Limiter} = State +) -> + case emqx_limiter_container:check_list(Needs, Limiter) of + {ok, Limiter2} -> + WhenOk(Data, Msgs, State#state{limiter = Limiter2}); + {pause, Time, Limiter2} -> + ?SLOG(debug, #{ + msg => "pause_time_dueto_rate_limit", + needs => Needs, + time_in_ms => Time + }), - Retry = #retry{ - types = [Type || {_, Type} <- Needs], - data = Data, - next = WhenOk - }, + Retry = #retry{ + types = [Type || {_, Type} <- Needs], + data = Data, + next = WhenOk + }, - Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), + Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), - TRef = start_timer(Time, limit_timeout), + TRef = start_timer(Time, limit_timeout), - {ok, State#state{ - limiter = Limiter3, - limiter_timer = TRef - }}; - {drop, Limiter2} -> - {ok, State#state{limiter = Limiter2}} - end; - _ -> - %% if there has a retry timer, - %% cache the operation and execute it after the retry is over - %% the maximum length of the cache queue is equal to the active_n - New = #pending_req{need = Needs, data = Data, next = WhenOk}, - {ok, State#state{limiter_buffer = queue:in(New, Cache)}} + {ok, State#state{ + limiter = Limiter3, + limiter_timer = TRef + }}; + {drop, Limiter2} -> + {ok, State#state{limiter = Limiter2}} end; -check_limiter(_, Data, WhenOk, Msgs, State) -> - WhenOk(Data, Msgs, State). +check_limiter( + Needs, + Data, + WhenOk, + _Msgs, + #state{limiter_buffer = Cache} = State +) -> + %% if there has a retry timer, + %% cache the operation and execute it after the retry is over + %% the maximum length of the cache queue is equal to the active_n + New = #pending_req{need = Needs, data = Data, next = WhenOk}, + {ok, State#state{limiter_buffer = queue:in(New, Cache)}}. %% try to perform a retry -spec retry_limiter(state()) -> _. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index ea02152a9..6a9101a0f 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -34,16 +34,18 @@ -export_type([container/0, check_result/0]). --type container() :: #{ - limiter_type() => undefined | limiter(), - %% the retry context of the limiter - retry_key() => - undefined - | retry_context() - | future(), - %% the retry context of the container - retry_ctx := undefined | any() -}. +-type container() :: + infinity + | #{ + limiter_type() => undefined | limiter(), + %% the retry context of the limiter + retry_key() => + undefined + | retry_context() + | future(), + %% the retry context of the container + retry_ctx := undefined | any() + }. -type future() :: pos_integer(). -type limiter_id() :: emqx_limiter_schema:limiter_id(). @@ -78,7 +80,20 @@ get_limiter_by_types(Id, Types, BucketCfgs) -> {ok, Limiter} = emqx_limiter_server:connect(Id, Type, BucketCfgs), add_new(Type, Limiter, Acc) end, - lists:foldl(Init, #{retry_ctx => undefined}, Types). + Container = lists:foldl(Init, #{retry_ctx => undefined}, Types), + case + lists:all( + fun(Type) -> + maps:get(Type, Container) =:= infinity + end, + Types + ) + of + true -> + infinity; + _ -> + Container + end. -spec add_new(limiter_type(), limiter(), container()) -> container(). add_new(Type, Limiter, Container) -> @@ -89,11 +104,15 @@ add_new(Type, Limiter, Container) -> %% @doc check the specified limiter -spec check(pos_integer(), limiter_type(), container()) -> check_result(). +check(_Need, _Type, infinity) -> + {ok, infinity}; check(Need, Type, Container) -> check_list([{Need, Type}], Container). %% @doc check multiple limiters -spec check_list(list({pos_integer(), limiter_type()}), container()) -> check_result(). +check_list(_Need, infinity) -> + {ok, infinity}; check_list([{Need, Type} | T], Container) -> Limiter = maps:get(Type, Container), case emqx_htb_limiter:check(Need, Limiter) of @@ -121,11 +140,15 @@ check_list([], Container) -> %% @doc retry the specified limiter -spec retry(limiter_type(), container()) -> check_result(). +retry(_Type, infinity) -> + {ok, infinity}; retry(Type, Container) -> retry_list([Type], Container). %% @doc retry multiple limiters -spec retry_list(list(limiter_type()), container()) -> check_result(). +retry_list(_Types, infinity) -> + {ok, infinity}; retry_list([Type | T], Container) -> Key = ?RETRY_KEY(Type), case Container of diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index faf62f98d..00fe545eb 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -90,7 +90,7 @@ listener :: {Type :: atom(), Name :: atom()}, %% Limiter - limiter :: maybe(container()), + limiter :: container(), %% cache operation when overload limiter_cache :: queue:queue(cache()), @@ -579,54 +579,61 @@ handle_timeout(TRef, TMsg, State) -> list(any()), state() ) -> state(). +check_limiter( + _Needs, + Data, + WhenOk, + Msgs, + #state{limiter = infinity} = State +) -> + WhenOk(Data, Msgs, State); check_limiter( Needs, Data, WhenOk, Msgs, - #state{ - limiter = Limiter, - limiter_timer = LimiterTimer, - limiter_cache = Cache - } = State + #state{limiter_timer = undefined, limiter = Limiter} = State ) -> - case LimiterTimer of - undefined -> - case emqx_limiter_container:check_list(Needs, Limiter) of - {ok, Limiter2} -> - WhenOk(Data, Msgs, State#state{limiter = Limiter2}); - {pause, Time, Limiter2} -> - ?SLOG(debug, #{ - msg => "pause_time_due_to_rate_limit", - needs => Needs, - time_in_ms => Time - }), + case emqx_limiter_container:check_list(Needs, Limiter) of + {ok, Limiter2} -> + WhenOk(Data, Msgs, State#state{limiter = Limiter2}); + {pause, Time, Limiter2} -> + ?SLOG(debug, #{ + msg => "pause_time_due_to_rate_limit", + needs => Needs, + time_in_ms => Time + }), - Retry = #retry{ - types = [Type || {_, Type} <- Needs], - data = Data, - next = WhenOk - }, + Retry = #retry{ + types = [Type || {_, Type} <- Needs], + data = Data, + next = WhenOk + }, - Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), + Limiter3 = emqx_limiter_container:set_retry_context(Retry, Limiter2), - TRef = start_timer(Time, limit_timeout), + TRef = start_timer(Time, limit_timeout), - enqueue( - {active, false}, - State#state{ - sockstate = blocked, - limiter = Limiter3, - limiter_timer = TRef - } - ); - {drop, Limiter2} -> - {ok, State#state{limiter = Limiter2}} - end; - _ -> - New = #cache{need = Needs, data = Data, next = WhenOk}, - State#state{limiter_cache = queue:in(New, Cache)} - end. + enqueue( + {active, false}, + State#state{ + sockstate = blocked, + limiter = Limiter3, + limiter_timer = TRef + } + ); + {drop, Limiter2} -> + {ok, State#state{limiter = Limiter2}} + end; +check_limiter( + Needs, + Data, + WhenOk, + _Msgs, + #state{limiter_cache = Cache} = State +) -> + New = #cache{need = Needs, data = Data, next = WhenOk}, + State#state{limiter_cache = queue:in(New, Cache)}. -spec retry_limiter(state()) -> state(). retry_limiter(#state{limiter = Limiter} = State) -> diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index f24c1c895..6b89227ab 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -38,8 +38,6 @@ init_per_suite(Config) -> ok = meck:new(emqx_cm, [passthrough, no_history, no_link]), ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end), ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), - %% Meck Limiter - ok = meck:new(emqx_htb_limiter, [passthrough, no_history, no_link]), %% Meck Pd ok = meck:new(emqx_pd, [passthrough, no_history, no_link]), %% Meck Metrics @@ -67,7 +65,6 @@ end_per_suite(_Config) -> ok = meck:unload(emqx_transport), catch meck:unload(emqx_channel), ok = meck:unload(emqx_cm), - ok = meck:unload(emqx_htb_limiter), ok = meck:unload(emqx_pd), ok = meck:unload(emqx_metrics), ok = meck:unload(emqx_hooks), @@ -421,6 +418,14 @@ t_ensure_rate_limit(_) -> {ok, [], State1} = emqx_connection:check_limiter([], [], WhenOk, [], st(#{limiter => Limiter})), ?assertEqual(Limiter, emqx_connection:info(limiter, State1)), + ok = meck:new(emqx_htb_limiter, [passthrough, no_history, no_link]), + + ok = meck:expect( + emqx_htb_limiter, + make_infinity_limiter, + fun() -> non_infinity end + ), + ok = meck:expect( emqx_htb_limiter, check, @@ -431,10 +436,10 @@ t_ensure_rate_limit(_) -> [], WhenOk, [], - st(#{limiter => Limiter}) + st(#{limiter => init_limiter()}) ), meck:unload(emqx_htb_limiter), - ok = meck:new(emqx_htb_limiter, [passthrough, no_history, no_link]), + ?assertNotEqual(undefined, emqx_connection:info(limiter_timer, State2)). t_activate_socket(_) -> @@ -707,7 +712,14 @@ init_limiter() -> limiter_cfg() -> Cfg = bucket_cfg(), - Client = #{ + Client = client_cfg(), + #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. + +bucket_cfg() -> + #{rate => infinity, initial => 0, burst => 0}. + +client_cfg() -> + #{ rate => infinity, initial => 0, burst => 0, @@ -715,11 +727,7 @@ limiter_cfg() -> divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force - }, - #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. - -bucket_cfg() -> - #{rate => infinity, initial => 0, burst => 0}. + }. add_bucket() -> Cfg = bucket_cfg(), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index 1ae23361e..813656e6a 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -443,7 +443,12 @@ t_websocket_info_deliver(_) -> t_websocket_info_timeout_limiter(_) -> Ref = make_ref(), - LimiterT = init_limiter(), + {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), + LimiterT = init_limiter(#{ + bytes => bucket_cfg(), + messages => bucket_cfg(), + client => #{bytes => client_cfg(Rate)} + }), Next = fun emqx_ws_connection:when_msg_in/3, Limiter = emqx_limiter_container:set_retry_context({retry, [], [], Next}, LimiterT), Event = {timeout, Ref, limit_timeout}, From 2f2f32ac7bffffeab9c409c5b3ee1b9fc5a634c3 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 10:52:30 +0800 Subject: [PATCH 040/194] chore: update changes --- changes/ce/perf-10487.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/perf-10487.en.md diff --git a/changes/ce/perf-10487.en.md b/changes/ce/perf-10487.en.md new file mode 100644 index 000000000..6f2b2d156 --- /dev/null +++ b/changes/ce/perf-10487.en.md @@ -0,0 +1 @@ +Optimize the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. From 7b51a49f84df0c5f96872edd658ad25b11625798 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 14:09:23 +0800 Subject: [PATCH 041/194] fix(limiter): remove the default limit of connect rate --- .../emqx_limiter/src/emqx_limiter_schema.erl | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index c762a0f1d..ae8529470 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -36,11 +36,11 @@ ]). -define(KILOBYTE, 1024). --define(BUCKET_KEYS, [ - {bytes, bucket_infinity}, - {messages, bucket_infinity}, - {connection, bucket_limit}, - {message_routing, bucket_infinity} +-define(LISTENER_BUCKET_KEYS, [ + bytes, + messages, + connection, + message_routing ]). -type limiter_type() :: @@ -132,10 +132,8 @@ fields(node_opts) -> ]; fields(client_fields) -> client_fields(types(), #{default => #{}}); -fields(bucket_infinity) -> +fields(bucket_opts) -> fields_of_bucket(<<"infinity">>); -fields(bucket_limit) -> - fields_of_bucket(<<"1000/s">>); fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, @@ -194,10 +192,9 @@ fields(client_opts) -> )} ]; fields(listener_fields) -> - composite_bucket_fields(?BUCKET_KEYS, listener_client_fields); + composite_bucket_fields(?LISTENER_BUCKET_KEYS, listener_client_fields); fields(listener_client_fields) -> - {Types, _} = lists:unzip(?BUCKET_KEYS), - client_fields(Types, #{required => false}); + client_fields(?LISTENER_BUCKET_KEYS, #{required => false}); fields(Type) -> simple_bucket_field(Type). @@ -205,10 +202,8 @@ desc(limiter) -> "Settings for the rate limiter."; desc(node_opts) -> "Settings for the limiter of the node level."; -desc(bucket_infinity) -> +desc(bucket_opts) -> "Settings for the bucket."; -desc(bucket_limit) -> - desc(bucket_infinity); desc(client_opts) -> "Settings for the client in bucket level."; desc(client_fields) -> @@ -360,7 +355,7 @@ apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). %% A bucket with only one type simple_bucket_field(Type) when is_atom(Type) -> - fields(bucket_infinity) ++ + fields(bucket_opts) ++ [ {client, ?HOCON( @@ -378,13 +373,13 @@ simple_bucket_field(Type) when is_atom(Type) -> composite_bucket_fields(Types, ClientRef) -> [ {Type, - ?HOCON(?R_REF(?MODULE, Opts), #{ + ?HOCON(?R_REF(?MODULE, bucket_opts), #{ desc => ?DESC(?MODULE, Type), required => false, importance => importance_of_type(Type), aliases => alias_of_type(Type) })} - || {Type, Opts} <- Types + || Type <- Types ] ++ [ {client, From 24cecae1f8fd5cbfb7025335a6a1dda2166c4222 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 14:15:45 +0800 Subject: [PATCH 042/194] chore: update changes --- changes/ce/perf-10490.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/perf-10490.en.md diff --git a/changes/ce/perf-10490.en.md b/changes/ce/perf-10490.en.md new file mode 100644 index 000000000..5c1c183a5 --- /dev/null +++ b/changes/ce/perf-10490.en.md @@ -0,0 +1 @@ +Remove the default limit of connect rate which used to be `1000/s` From 0b1a2dd1939428af2131490007a492c598363898 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 24 Apr 2023 14:40:30 +0800 Subject: [PATCH 043/194] feat: rename etcd.ssl to etcd.ssl_options --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 234690374..03cd36522 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index f3f014321..abccca9fb 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -335,11 +335,12 @@ fields(cluster_etcd) -> desc => ?DESC(cluster_etcd_node_ttl) } )}, - {"ssl", + {"ssl_options", sc( ?R_REF(emqx_schema, "ssl_client_opts"), #{ desc => ?DESC(cluster_etcd_ssl), + alias => [ssl], 'readOnly' => true } )} From dacf92c4ab892867467d31f0d6cd947cd689a791 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 24 Apr 2023 14:45:53 +0800 Subject: [PATCH 044/194] chore: rename etcd.ssl changelog --- changes/ce/feat-10491.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10491.en.md diff --git a/changes/ce/feat-10491.en.md b/changes/ce/feat-10491.en.md new file mode 100644 index 000000000..e1c38b6bb --- /dev/null +++ b/changes/ce/feat-10491.en.md @@ -0,0 +1 @@ +Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file. From db0c951e3013e2e59adc7dd101cd70ebb04ad9f3 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 24 Apr 2023 15:27:42 +0800 Subject: [PATCH 045/194] feat: don't do rpc call to check deprecated file --- apps/emqx/priv/bpapi.versions | 1 - apps/emqx_conf/src/emqx_conf_app.erl | 12 +- .../src/proto/emqx_conf_proto_v2.erl | 4 - .../src/proto/emqx_conf_proto_v3.erl | 114 ------------------ 4 files changed, 7 insertions(+), 124 deletions(-) delete mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 11bd4aa77..db4765e3f 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -11,7 +11,6 @@ {emqx_cm,1}. {emqx_conf,1}. {emqx_conf,2}. -{emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index fd0a56853..fbfb97a79 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -66,7 +66,8 @@ get_override_config_file() -> conf => Conf, tnx_id => TnxId, node => Node, - has_deprecated_file => HasDeprecateFile + has_deprecated_file => HasDeprecateFile, + release => emqx_app:get_release() } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of @@ -180,6 +181,8 @@ copy_override_conf_from_core_node() -> msg => "copy_cluster_conf_from_core_node_success", node => Node, has_deprecated_file => HasDeprecatedFile, + local_release => emqx_app:get_release(), + remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"), data_dir => emqx:data_dir(), tnx_id => TnxId }), @@ -228,13 +231,12 @@ sync_data_from_node(Node) -> error(Error) end. -has_deprecated_file(#{node := Node} = Info) -> +has_deprecated_file(#{conf := Conf} = Info) -> case maps:find(has_deprecated_file, Info) of {ok, HasDeprecatedFile} -> HasDeprecatedFile; error -> %% The old version don't have emqx_config:has_deprecated_file/0 - DataDir = emqx_conf_proto_v2:get_config(Node, [node, data_dir]), - File = filename:join([DataDir, "configs", "cluster-override.conf"]), - emqx_conf_proto_v3:file_exist(Node, File) + %% Conf is not empty if deprecated file is found. + Conf =/= #{} end. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index 3bcf532f6..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -20,7 +20,6 @@ -export([ introduced_in/0, - deprecated_since/0, sync_data_from_node/1, get_config/2, get_config/3, @@ -42,9 +41,6 @@ introduced_in() -> "5.0.1". -deprecated_since() -> - "5.0.23". - -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl deleted file mode 100644 index 802436f98..000000000 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl +++ /dev/null @@ -1,114 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_conf_proto_v3). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - sync_data_from_node/1, - get_config/2, - get_config/3, - get_all/1, - - update/3, - update/4, - remove_config/2, - remove_config/3, - - reset/2, - reset/3, - - get_override_config_file/1, - file_exist/2 -]). - --include_lib("emqx/include/bpapi.hrl"). - -introduced_in() -> - "5.0.24". - --spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). -sync_data_from_node(Node) -> - rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. - --spec get_config(node(), emqx_utils_maps:config_key_path()) -> - term() | emqx_rpc:badrpc(). -get_config(Node, KeyPath) -> - rpc:call(Node, emqx, get_config, [KeyPath]). - --spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> - term() | emqx_rpc:badrpc(). -get_config(Node, KeyPath, Default) -> - rpc:call(Node, emqx, get_config, [KeyPath, Default]). - --spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). -get_all(KeyPath) -> - rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). - --spec update( - update_config_key_path(), - emqx_config:update_request(), - emqx_config:update_opts() -) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -update(KeyPath, UpdateReq, Opts) -> - emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). - --spec update( - node(), - update_config_key_path(), - emqx_config:update_request(), - emqx_config:update_opts() -) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -update(Node, KeyPath, UpdateReq, Opts) -> - rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). - --spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -remove_config(KeyPath, Opts) -> - emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). - --spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -remove_config(Node, KeyPath, Opts) -> - rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). - --spec reset(update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -reset(KeyPath, Opts) -> - emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). - --spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -reset(Node, KeyPath, Opts) -> - rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). - --spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). -get_override_config_file(Nodes) -> - rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). - --spec file_exist(node(), string()) -> emqx_rpc:badrpc() | boolean(). -file_exist(Node, File) -> - rpc:call(Node, filelib, is_regular, [File], 5000). From 275967a49f54dc68d2a63f8023d31dd70ca11239 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 24 Apr 2023 16:13:42 +0800 Subject: [PATCH 046/194] chore: remove dashboard's default username from emqx.conf --- apps/emqx_dashboard/etc/emqx_dashboard.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 856779500..67e3f61ec 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -2,6 +2,4 @@ dashboard { listeners.http { bind = 18083 } - default_username = "admin" - default_password = "public" } From 7ce04358c411ba85e9fdb6f5154d56daaf89a8ab Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 17:28:33 +0800 Subject: [PATCH 047/194] fix(Dynamo): fix DynamoDB bridge status check error --- .../emqx_ee_connector/src/emqx_ee_connector_dynamo.erl | 2 +- .../src/emqx_ee_connector_dynamo_client.erl | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 2170827d6..01554f90a 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -34,7 +34,7 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). -define(DYNAMO_HOST_OPTIONS, #{ - default_port => 8000 + default_port => 80 }). %%===================================================================== diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl index e0d8ee4bf..0340655b4 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl @@ -71,8 +71,14 @@ init(#{ {ok, #{}}. handle_call(is_connected, _From, State) -> - _ = erlcloud_ddb2:list_tables(), - {reply, true, State}; + IsConnected = + case erlcloud_ddb2:list_tables([{limit, 1}]) of + {ok, _} -> + true; + _ -> + false + end, + {reply, IsConnected, State}; handle_call({query, Table, Query, Templates}, _From, State) -> Result = do_query(Table, Query, Templates), {reply, Result, State}; From feeb3df994c992f4246c0bca5613346931822e0d Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 18:09:17 +0800 Subject: [PATCH 048/194] fix(api): add limiter API back which deleted by mistake --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index af203dfe9..bc9aaf768 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -42,6 +42,7 @@ <<"alarm">>, <<"sys_topics">>, <<"sysmon">>, + <<"limiter">>, <<"log">>, <<"persistent_session_store">>, <<"zones">> From 6110aad23f135e819197c0600c48ce688118d0f6 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Apr 2023 18:16:28 +0800 Subject: [PATCH 049/194] chore: bump version && update changes --- apps/emqx_management/src/emqx_management.app.src | 2 +- changes/ce/fix-10495.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-10495.en.md diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f423213af..ec282b60b 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.19"}, + {vsn, "5.0.20"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/changes/ce/fix-10495.en.md b/changes/ce/fix-10495.en.md new file mode 100644 index 000000000..222f3dd5a --- /dev/null +++ b/changes/ce/fix-10495.en.md @@ -0,0 +1 @@ +Add the limiter API `/configs/limiter` which was deleted by mistake back. From 8cfb24b5b435936759b98fc0e6b53576755637bf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 6 Apr 2023 15:04:44 -0300 Subject: [PATCH 050/194] docs(kafka_bridge): minor fixes to license and readme Fixes https://emqx.atlassian.net/browse/EMQX-9481 --- LICENSE | 6 +++++- apps/emqx_bridge_kafka/README.md | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/LICENSE b/LICENSE index 2a081b135..4ed190bda 100644 --- a/LICENSE +++ b/LICENSE @@ -4,4 +4,8 @@ For EMQX: Apache License 2.0, see APL.txt, which applies to all source files except for lib-ee sub-directory. For EMQX Enterprise (since version 5.0): Business Source License 1.1, -see lib-ee/BSL.txt, which applies to source code in lib-ee sub-directory. +see lib-ee/BSL.txt, which applies to source code in lib-ee +sub-directory and some of the apps under the apps directory. + +Source code under apps that uses BSL License: +- apps/emqx_bridge_kafka diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md index 72cbeecc6..80978ff10 100644 --- a/apps/emqx_bridge_kafka/README.md +++ b/apps/emqx_bridge_kafka/README.md @@ -10,10 +10,21 @@ workers from `emqx_resource`. It implements the connection management and interaction without need for a separate connector app, since it's not used by authentication and authorization applications. -## Contributing +# Documentation links + +For more information on Apache Kafka, please see its [official +site](https://kafka.apache.org/). + +# Configurations + +Please see [our official +documentation](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-kafka.html) +for more detailed info. + +# Contributing Please see our [contributing.md](../../CONTRIBUTING.md). -## License +# License See [BSL](./BSL.txt). From 26883eec02e52ee5c35afdf46f47803085c2b978 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 13 Apr 2023 15:38:18 -0300 Subject: [PATCH 051/194] test(kafka): fix innocuous test assertion --- .../test/emqx_bridge_kafka_impl_consumer_SUITE.erl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 08fbf5e15..bb8930280 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1156,11 +1156,12 @@ t_start_and_consume_ok(Config) -> ), %% Check that the bridge probe API doesn't leak atoms. - ProbeRes = probe_bridge_api(Config), - ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + ProbeRes0 = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), AtomsBefore = erlang:system_info(atom_count), %% Probe again; shouldn't have created more atoms. - ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + ProbeRes1 = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes1), AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), @@ -1259,11 +1260,12 @@ t_multiple_topic_mappings(Config) -> {ok, _} = snabbkaffe:receive_events(SRef0), %% Check that the bridge probe API doesn't leak atoms. - ProbeRes = probe_bridge_api(Config), - ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + ProbeRes0 = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), AtomsBefore = erlang:system_info(atom_count), %% Probe again; shouldn't have created more atoms. - ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes), + ProbeRes1 = probe_bridge_api(Config), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes1), AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), From 4bcfbea056719be91158be4d7eb2eff6299c83f4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 13 Apr 2023 16:26:03 -0300 Subject: [PATCH 052/194] refactor(kafka_consumer): follow up refactoring requested from previous pull request Follow up from https://github.com/emqx/emqx/pull/10273 --- .../src/emqx_bridge_kafka_impl_consumer.erl | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index fdfa3300c..c549b3467 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -179,7 +179,12 @@ on_get_status(_InstanceID, State) -> kafka_client_id := ClientID, kafka_topics := KafkaTopics } = State, - do_get_status(State, ClientID, KafkaTopics, SubscriberId). + case do_get_status(ClientID, KafkaTopics, SubscriberId) of + {disconnected, Message} -> + {disconnected, State, Message}; + Res -> + Res + end. %%------------------------------------------------------------------------------------- %% `brod_group_subscriber' API @@ -376,41 +381,41 @@ stop_client(ClientID) -> ), ok. -do_get_status(State, ClientID, [KafkaTopic | RestTopics], SubscriberId) -> +do_get_status(ClientID, [KafkaTopic | RestTopics], SubscriberId) -> case brod:get_partitions_count(ClientID, KafkaTopic) of {ok, NPartitions} -> - case do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) of - connected -> do_get_status(State, ClientID, RestTopics, SubscriberId); + case do_get_topic_status(ClientID, KafkaTopic, SubscriberId, NPartitions) of + connected -> do_get_status(ClientID, RestTopics, SubscriberId); disconnected -> disconnected end; {error, {client_down, Context}} -> case infer_client_error(Context) of auth_error -> Message = "Authentication error. " ++ ?CLIENT_DOWN_MESSAGE, - {disconnected, State, Message}; + {disconnected, Message}; {auth_error, Message0} -> Message = binary_to_list(Message0) ++ "; " ++ ?CLIENT_DOWN_MESSAGE, - {disconnected, State, Message}; + {disconnected, Message}; connection_refused -> Message = "Connection refused. " ++ ?CLIENT_DOWN_MESSAGE, - {disconnected, State, Message}; + {disconnected, Message}; _ -> - {disconnected, State, ?CLIENT_DOWN_MESSAGE} + {disconnected, ?CLIENT_DOWN_MESSAGE} end; {error, leader_not_available} -> Message = "Leader connection not available. Please check the Kafka topic used," " the connection parameters and Kafka cluster health", - {disconnected, State, Message}; + {disconnected, Message}; _ -> disconnected end; -do_get_status(_State, _ClientID, _KafkaTopics = [], _SubscriberId) -> +do_get_status(_ClientID, _KafkaTopics = [], _SubscriberId) -> connected. --spec do_get_status1(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> +-spec do_get_topic_status(brod:client_id(), binary(), subscriber_id(), pos_integer()) -> connected | disconnected. -do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) -> +do_get_topic_status(ClientID, KafkaTopic, SubscriberId, NPartitions) -> Results = lists:map( fun(N) -> From dc480323092ccf2e2259c13ccba3598d59753b54 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 12 Apr 2023 10:21:57 -0300 Subject: [PATCH 053/194] feat(schema): add support for schemes in server parser/validator --- apps/emqx/src/emqx_schema.erl | 82 +++++++++++-- apps/emqx/test/emqx_schema_tests.erl | 167 ++++++++++++++++++++++++++- 2 files changed, 240 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 272a1d0cd..248fdad7f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -42,7 +42,12 @@ -type ip_port() :: tuple() | integer(). -type cipher() :: map(). -type port_number() :: 1..65536. --type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. +-type server_parse_option() :: #{ + default_port => port_number(), + no_port => boolean(), + supported_schemes => [string()], + default_scheme => string() +}. -type url() :: binary(). -type json_binary() :: binary(). @@ -2896,7 +2901,10 @@ servers_validator(Opts, Required) -> %% `no_port': by default it's `false', when set to `true', %% a `throw' exception is raised if the port is found. -spec parse_server(undefined | string() | binary(), server_parse_option()) -> - {string(), port_number()}. + string() + | {string(), port_number()} + | {string(), string()} + | {string(), string(), port_number()}. parse_server(Str, Opts) -> case parse_servers(Str, Opts) of undefined -> @@ -2910,7 +2918,12 @@ parse_server(Str, Opts) -> %% @doc Parse comma separated `host[:port][,host[:port]]' endpoints %% into a list of `{Host, Port}' tuples or just `Host' string. -spec parse_servers(undefined | string() | binary(), server_parse_option()) -> - [{string(), port_number()}]. + [ + string() + | {string(), port_number()} + | {string(), string()} + | {string(), string(), port_number()} + ]. parse_servers(undefined, _Opts) -> %% should not parse 'undefined' as string, %% not to throw exception either, @@ -2956,6 +2969,9 @@ split_host_port(Str) -> do_parse_server(Str, Opts) -> DefaultPort = maps:get(default_port, Opts, undefined), NotExpectingPort = maps:get(no_port, Opts, false), + DefaultScheme = maps:get(default_scheme, Opts, undefined), + SupportedSchemes = maps:get(supported_schemes, Opts, []), + NotExpectingScheme = (not is_list(DefaultScheme)) andalso length(SupportedSchemes) =:= 0, case is_integer(DefaultPort) andalso NotExpectingPort of true -> %% either provide a default port from schema, @@ -2964,24 +2980,74 @@ do_parse_server(Str, Opts) -> false -> ok end, + case is_list(DefaultScheme) andalso (not lists:member(DefaultScheme, SupportedSchemes)) of + true -> + %% inconsistent schema + error("bad_schema"); + false -> + ok + end, %% do not split with space, there should be no space allowed between host and port case string:tokens(Str, ":") of - [Hostname, Port] -> + [Scheme, "//" ++ Hostname, Port] -> NotExpectingPort andalso throw("not_expecting_port_number"), - {check_hostname(Hostname), parse_port(Port)}; - [Hostname] -> + NotExpectingScheme andalso throw("not_expecting_scheme"), + {check_scheme(Scheme, Opts), check_hostname(Hostname), parse_port(Port)}; + [Scheme, "//" ++ Hostname] -> + NotExpectingScheme andalso throw("not_expecting_scheme"), case is_integer(DefaultPort) of true -> - {check_hostname(Hostname), DefaultPort}; + {check_scheme(Scheme, Opts), check_hostname(Hostname), DefaultPort}; false when NotExpectingPort -> - check_hostname(Hostname); + {check_scheme(Scheme, Opts), check_hostname(Hostname)}; false -> throw("missing_port_number") end; + [Hostname, Port] -> + NotExpectingPort andalso throw("not_expecting_port_number"), + case is_list(DefaultScheme) of + false -> + {check_hostname(Hostname), parse_port(Port)}; + true -> + {DefaultScheme, check_hostname(Hostname), parse_port(Port)} + end; + [Hostname] -> + case is_integer(DefaultPort) orelse NotExpectingPort of + true -> + ok; + false -> + throw("missing_port_number") + end, + case is_list(DefaultScheme) orelse NotExpectingScheme of + true -> + ok; + false -> + throw("missing_scheme") + end, + case {is_integer(DefaultPort), is_list(DefaultScheme)} of + {true, true} -> + {DefaultScheme, check_hostname(Hostname), DefaultPort}; + {true, false} -> + {check_hostname(Hostname), DefaultPort}; + {false, true} -> + {DefaultScheme, check_hostname(Hostname)}; + {false, false} -> + check_hostname(Hostname) + end; _ -> throw("bad_host_port") end. +check_scheme(Str, Opts) -> + SupportedSchemes = maps:get(supported_schemes, Opts, []), + IsSupported = lists:member(Str, SupportedSchemes), + case IsSupported of + true -> + Str; + false -> + throw("unsupported_scheme") + end. + check_hostname(Str) -> %% not intended to use inet_parse:domain here %% only checking space because it interferes the parsing diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 5176f4fad..c13dc8055 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -350,7 +350,7 @@ parse_server_test_() -> ) ), ?T( - "multiple servers wihtout port, mixed list(binary|string)", + "multiple servers without port, mixed list(binary|string)", ?assertEqual( ["host1", "host2"], Parse2([<<"host1">>, "host2"], #{no_port => true}) @@ -447,6 +447,171 @@ parse_server_test_() -> "bad_schema", emqx_schema:parse_server("whatever", #{default_port => 10, no_port => true}) ) + ), + ?T( + "scheme, hostname and port", + ?assertEqual( + {"pulsar+ssl", "host", 6651}, + emqx_schema:parse_server( + "pulsar+ssl://host:6651", + #{ + default_port => 6650, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "scheme and hostname, default port", + ?assertEqual( + {"pulsar", "host", 6650}, + emqx_schema:parse_server( + "pulsar://host", + #{ + default_port => 6650, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "scheme and hostname, no port", + ?assertEqual( + {"pulsar", "host"}, + emqx_schema:parse_server( + "pulsar://host", + #{ + no_port => true, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "scheme and hostname, missing port", + ?assertThrow( + "missing_port_number", + emqx_schema:parse_server( + "pulsar://host", + #{ + no_port => false, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "hostname, default scheme, no default port", + ?assertEqual( + {"pulsar", "host"}, + emqx_schema:parse_server( + "host", + #{ + default_scheme => "pulsar", + no_port => true, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "hostname, default scheme, default port", + ?assertEqual( + {"pulsar", "host", 6650}, + emqx_schema:parse_server( + "host", + #{ + default_port => 6650, + default_scheme => "pulsar", + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "just hostname, expecting missing scheme", + ?assertThrow( + "missing_scheme", + emqx_schema:parse_server( + "host", + #{ + no_port => true, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "hostname, default scheme, defined port", + ?assertEqual( + {"pulsar", "host", 6651}, + emqx_schema:parse_server( + "host:6651", + #{ + default_port => 6650, + default_scheme => "pulsar", + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "inconsistent scheme opts", + ?assertError( + "bad_schema", + emqx_schema:parse_server( + "pulsar+ssl://host:6651", + #{ + default_port => 6650, + default_scheme => "something", + supported_schemes => ["not", "supported"] + } + ) + ) + ), + ?T( + "hostname, default scheme, defined port", + ?assertEqual( + {"pulsar", "host", 6651}, + emqx_schema:parse_server( + "host:6651", + #{ + default_port => 6650, + default_scheme => "pulsar", + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) + ), + ?T( + "unsupported scheme", + ?assertThrow( + "unsupported_scheme", + emqx_schema:parse_server( + "pulsar+quic://host:6651", + #{ + default_port => 6650, + supported_schemes => ["pulsar"] + } + ) + ) + ), + ?T( + "multiple hostnames with schemes (1)", + ?assertEqual( + [ + {"pulsar", "host", 6649}, + {"pulsar+ssl", "other.host", 6651}, + {"pulsar", "yet.another", 6650} + ], + emqx_schema:parse_servers( + "pulsar://host:6649, pulsar+ssl://other.host:6651,pulsar://yet.another", + #{ + default_port => 6650, + supported_schemes => ["pulsar", "pulsar+ssl"] + } + ) + ) ) ]. From f6da118ebd16d445f8ae3df1b597e7d87e0a3084 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 17 Apr 2023 17:51:54 -0300 Subject: [PATCH 054/194] test: fix flaky test --- apps/emqx/test/emqx_connection_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index f24c1c895..6f0e3eb20 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -495,6 +495,7 @@ t_get_conn_info(_) -> end). t_oom_shutdown(init, Config) -> + ok = snabbkaffe:stop(), ok = snabbkaffe:start_trace(), ok = meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), meck:expect( From ad4be08bb2bac7c4d730070dc268fc2b54afccc0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 12 Apr 2023 10:22:36 -0300 Subject: [PATCH 055/194] feat: implement Pulsar Producer bridge (e5.0) Fixes https://emqx.atlassian.net/browse/EMQX-8398 --- .../docker-compose-pulsar-tcp.yaml | 32 + .ci/docker-compose-file/toxiproxy.json | 12 + LICENSE | 1 + apps/emqx/test/emqx_test_janitor.erl | 24 +- apps/emqx_bridge/src/emqx_bridge.erl | 3 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 2 + apps/emqx_bridge_pulsar/.gitignore | 19 + apps/emqx_bridge_pulsar/BSL.txt | 94 ++ apps/emqx_bridge_pulsar/README.md | 30 + apps/emqx_bridge_pulsar/docker-ct | 2 + .../etc/emqx_bridge_pulsar.conf | 0 .../include/emqx_bridge_pulsar.hrl | 14 + apps/emqx_bridge_pulsar/rebar.config | 13 + .../src/emqx_bridge_pulsar.app.src | 15 + .../src/emqx_bridge_pulsar.erl | 228 +++++ .../src/emqx_bridge_pulsar_app.erl | 14 + .../src/emqx_bridge_pulsar_impl_producer.erl | 396 +++++++++ .../src/emqx_bridge_pulsar_sup.erl | 33 + ...emqx_bridge_pulsar_impl_producer_SUITE.erl | 819 ++++++++++++++++++ .../pulsar_echo_consumer.erl | 25 + .../src/emqx_resource_manager.erl | 7 +- changes/ee/feat-10378.en.md | 1 + .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 4 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 24 +- mix.exs | 4 +- rebar.config.erl | 1 + rel/i18n/emqx_bridge_pulsar.hocon | 176 ++++ rel/i18n/zh/emqx_bridge_pulsar.hocon | 173 ++++ scripts/ct/run.sh | 5 +- 29 files changed, 2160 insertions(+), 11 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-pulsar-tcp.yaml create mode 100644 apps/emqx_bridge_pulsar/.gitignore create mode 100644 apps/emqx_bridge_pulsar/BSL.txt create mode 100644 apps/emqx_bridge_pulsar/README.md create mode 100644 apps/emqx_bridge_pulsar/docker-ct create mode 100644 apps/emqx_bridge_pulsar/etc/emqx_bridge_pulsar.conf create mode 100644 apps/emqx_bridge_pulsar/include/emqx_bridge_pulsar.hrl create mode 100644 apps/emqx_bridge_pulsar/rebar.config create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl create mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl create mode 100644 apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl create mode 100644 apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE_data/pulsar_echo_consumer.erl create mode 100644 changes/ee/feat-10378.en.md create mode 100644 rel/i18n/emqx_bridge_pulsar.hocon create mode 100644 rel/i18n/zh/emqx_bridge_pulsar.hocon diff --git a/.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml b/.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml new file mode 100644 index 000000000..926000ae4 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml @@ -0,0 +1,32 @@ +version: '3' + +services: + pulsar: + container_name: pulsar + image: apachepulsar/pulsar:2.11.0 + # ports: + # - 6650:6650 + # - 8080:8080 + networks: + emqx_bridge: + volumes: + - ../../apps/emqx/etc/certs/cert.pem:/etc/certs/server.pem + - ../../apps/emqx/etc/certs/key.pem:/etc/certs/key.pem + - ../../apps/emqx/etc/certs/cacert.pem:/etc/certs/ca.pem + restart: always + command: + - bash + - "-c" + - | + sed -i 's/^advertisedAddress=/#advertisedAddress=/' conf/standalone.conf + sed -ie 's/^brokerServicePort=.*/brokerServicePort=6649/' conf/standalone.conf + sed -i 's/^bindAddress=/#bindAddress=/' conf/standalone.conf + sed -i 's#^bindAddresses=#bindAddresses=plain:pulsar://0.0.0.0:6650,ssl:pulsar+ssl://0.0.0.0:6651,toxiproxy:pulsar://0.0.0.0:6652,toxiproxy_ssl:pulsar+ssl://0.0.0.0:6653#' conf/standalone.conf + sed -i 's#^advertisedAddress=#advertisedAddress=plain:pulsar://pulsar:6650,ssl:pulsar+ssl://pulsar:6651,toxiproxy:pulsar://toxiproxy:6652,toxiproxy_ssl:pulsar+ssl://toxiproxy:6653#' conf/standalone.conf + sed -i 's#^tlsCertificateFilePath=#tlsCertificateFilePath=/etc/certs/server.pem#' conf/standalone.conf + sed -i 's#^tlsTrustCertsFilePath=#tlsTrustCertsFilePath=/etc/certs/ca.pem#' conf/standalone.conf + sed -i 's#^tlsKeyFilePath=#tlsKeyFilePath=/etc/certs/key.pem#' conf/standalone.conf + sed -i 's#^tlsProtocols=#tlsProtocols=TLSv1.3,TLSv1.2#' conf/standalone.conf + sed -i 's#^tlsCiphers=#tlsCiphers=TLS_AES_256_GCM_SHA384#' conf/standalone.conf + echo 'advertisedListeners=plain:pulsar://pulsar:6650,ssl:pulsar+ssl://pulsar:6651,toxiproxy:pulsar://toxiproxy:6652,toxiproxy_ssl:pulsar+ssl://toxiproxy:6653' >> conf/standalone.conf + bin/pulsar standalone -nfw -nss diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index f6b31da4c..9cefcb808 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -107,5 +107,17 @@ "listen": "0.0.0.0:4242", "upstream": "opents:4242", "enabled": true + }, + { + "name": "pulsar_plain", + "listen": "0.0.0.0:6652", + "upstream": "pulsar:6652", + "enabled": true + }, + { + "name": "pulsar_tls", + "listen": "0.0.0.0:6653", + "upstream": "pulsar:6653", + "enabled": true } ] diff --git a/LICENSE b/LICENSE index 4ed190bda..68bb18ce3 100644 --- a/LICENSE +++ b/LICENSE @@ -9,3 +9,4 @@ sub-directory and some of the apps under the apps directory. Source code under apps that uses BSL License: - apps/emqx_bridge_kafka +- apps/emqx_bridge_pulsar diff --git a/apps/emqx/test/emqx_test_janitor.erl b/apps/emqx/test/emqx_test_janitor.erl index c3f82a3e1..041b03fa7 100644 --- a/apps/emqx/test/emqx_test_janitor.erl +++ b/apps/emqx/test/emqx_test_janitor.erl @@ -60,12 +60,12 @@ init(Parent) -> {ok, #{callbacks => [], owner => Parent}}. terminate(_Reason, #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> catch Fun() end, Callbacks). + do_terminate(Callbacks). handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) -> {reply, ok, State#{callbacks := [Callback | Callbacks]}}; handle_call(terminate, _From, State = #{callbacks := Callbacks}) -> - lists:foreach(fun(Fun) -> catch Fun() end, Callbacks), + do_terminate(Callbacks), {stop, normal, ok, State}; handle_call(_Req, _From, State) -> {reply, error, State}. @@ -77,3 +77,23 @@ handle_info({'EXIT', Parent, _Reason}, State = #{owner := Parent}) -> {stop, normal, State}; handle_info(_Msg, State) -> {noreply, State}. + +%%---------------------------------------------------------------------------------- +%% Internal fns +%%---------------------------------------------------------------------------------- + +do_terminate(Callbacks) -> + lists:foreach( + fun(Fun) -> + try + Fun() + catch + K:E:S -> + ct:pal("error executing callback ~p: ~p", [Fun, {K, E}]), + ct:pal("stacktrace: ~p", [S]), + ok + end + end, + Callbacks + ), + ok. diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 08b8222f2..fd4e16263 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -70,7 +70,8 @@ T == dynamo; T == rocketmq; T == cassandra; - T == sqlserver + T == sqlserver; + T == pulsar_producer ). load() -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 1ad024c40..da98b073e 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -340,6 +340,8 @@ parse_confs(Type, Name, Conf) when ?IS_INGRESS_BRIDGE(Type) -> %% to hocon; keeping this as just `kafka' for backwards compatibility. parse_confs(<<"kafka">> = _Type, Name, Conf) -> Conf#{bridge_name => Name}; +parse_confs(<<"pulsar_producer">> = _Type, Name, Conf) -> + Conf#{bridge_name => Name}; parse_confs(_Type, _Name, Conf) -> Conf. diff --git a/apps/emqx_bridge_pulsar/.gitignore b/apps/emqx_bridge_pulsar/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_bridge_pulsar/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_bridge_pulsar/BSL.txt b/apps/emqx_bridge_pulsar/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_pulsar/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_pulsar/README.md b/apps/emqx_bridge_pulsar/README.md new file mode 100644 index 000000000..09e17d8bb --- /dev/null +++ b/apps/emqx_bridge_pulsar/README.md @@ -0,0 +1,30 @@ +# Pulsar Data Integration Bridge + +This application houses the Pulsar Producer data integration bridge +for EMQX Enterprise Edition. It provides the means to connect to +Pulsar and publish messages to it. + +Currently, our Pulsar Producer library has its own `replayq` buffering +implementation, so this bridge does not require buffer workers from +`emqx_resource`. It implements the connection management and +interaction without need for a separate connector app, since it's not +used by authentication and authorization applications. + +# Documentation links + +For more information on Apache Pulsar, please see its [official +site](https://pulsar.apache.org/). + +# Configurations + +Please see [our official +documentation](https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-pulsar.html) +for more detailed info. + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +# License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_pulsar/docker-ct b/apps/emqx_bridge_pulsar/docker-ct new file mode 100644 index 000000000..6324bb4f7 --- /dev/null +++ b/apps/emqx_bridge_pulsar/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +pulsar diff --git a/apps/emqx_bridge_pulsar/etc/emqx_bridge_pulsar.conf b/apps/emqx_bridge_pulsar/etc/emqx_bridge_pulsar.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_pulsar/include/emqx_bridge_pulsar.hrl b/apps/emqx_bridge_pulsar/include/emqx_bridge_pulsar.hrl new file mode 100644 index 000000000..5ee87e48f --- /dev/null +++ b/apps/emqx_bridge_pulsar/include/emqx_bridge_pulsar.hrl @@ -0,0 +1,14 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_BRIDGE_PULSAR_HRL). +-define(EMQX_BRIDGE_PULSAR_HRL, true). + +-define(PULSAR_HOST_OPTIONS, #{ + default_port => 6650, + default_scheme => "pulsar", + supported_schemes => ["pulsar", "pulsar+ssl"] +}). + +-endif. diff --git a/apps/emqx_bridge_pulsar/rebar.config b/apps/emqx_bridge_pulsar/rebar.config new file mode 100644 index 000000000..3b9ae417d --- /dev/null +++ b/apps/emqx_bridge_pulsar/rebar.config @@ -0,0 +1,13 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ {pulsar, {git, "https://github.com/emqx/pulsar-client-erl.git", {tag, "0.8.0"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_bridge_pulsar]} +]}. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src new file mode 100644 index 000000000..cd89f6867 --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -0,0 +1,15 @@ +{application, emqx_bridge_pulsar, [ + {description, "EMQX Pulsar Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_bridge_pulsar_app, []}}, + {applications, [ + kernel, + stdlib, + pulsar + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl new file mode 100644 index 000000000..a3e50054e --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -0,0 +1,228 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_pulsar). + +-include("emqx_bridge_pulsar.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +%% hocon_schema API +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). +%% emqx_ee_bridge "unofficial" API +-export([conn_bridge_examples/1]). + +%%------------------------------------------------------------------------------------------------- +%% `hocon_schema' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> + "bridge_pulsar". + +roots() -> + []. + +fields(pulsar_producer) -> + fields(config) ++ fields(producer_opts); +fields(config) -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {servers, + mk( + binary(), + #{ + required => true, + desc => ?DESC("servers"), + validator => emqx_schema:servers_validator( + ?PULSAR_HOST_OPTIONS, _Required = true + ) + } + )}, + {authentication, + mk(hoconsc:union([none, ref(auth_basic), ref(auth_token)]), #{ + default => none, desc => ?DESC("authentication") + })} + ] ++ emqx_connector_schema_lib:ssl_fields(); +fields(producer_opts) -> + [ + {batch_size, + mk( + pos_integer(), + #{default => 100, desc => ?DESC("producer_batch_size")} + )}, + {compression, + mk( + hoconsc:enum([no_compression, snappy, zlib]), + #{default => no_compression, desc => ?DESC("producer_compression")} + )}, + {send_buffer, + mk(emqx_schema:bytesize(), #{ + default => <<"1MB">>, desc => ?DESC("producer_send_buffer") + })}, + {sync_timeout, + mk(emqx_schema:duration_ms(), #{ + default => <<"3s">>, desc => ?DESC("producer_sync_timeout") + })}, + {retention_period, + mk( + hoconsc:union([infinity, emqx_schema:duration_ms()]), + #{default => infinity, desc => ?DESC("producer_retention_period")} + )}, + {max_batch_bytes, + mk( + emqx_schema:bytesize(), + #{default => <<"900KB">>, desc => ?DESC("producer_max_batch_bytes")} + )}, + {local_topic, mk(binary(), #{required => false, desc => ?DESC("producer_local_topic")})}, + {pulsar_topic, mk(binary(), #{required => true, desc => ?DESC("producer_pulsar_topic")})}, + {strategy, + mk( + hoconsc:enum([random, roundrobin, first_key_dispatch]), + #{default => random, desc => ?DESC("producer_strategy")} + )}, + {buffer, mk(ref(producer_buffer), #{required => false, desc => ?DESC("producer_buffer")})}, + {message, + mk(ref(producer_pulsar_message), #{ + required => false, desc => ?DESC("producer_message_opts") + })}, + {resource_opts, + mk( + ref(producer_resource_opts), + #{ + required => false, + desc => ?DESC(emqx_resource_schema, "creation_opts") + } + )} + ]; +fields(producer_buffer) -> + [ + {mode, + mk( + hoconsc:enum([memory, disk, hybrid]), + #{default => memory, desc => ?DESC("buffer_mode")} + )}, + {per_partition_limit, + mk( + emqx_schema:bytesize(), + #{default => <<"2GB">>, desc => ?DESC("buffer_per_partition_limit")} + )}, + {segment_bytes, + mk( + emqx_schema:bytesize(), + #{default => <<"100MB">>, desc => ?DESC("buffer_segment_bytes")} + )}, + {memory_overload_protection, + mk(boolean(), #{ + default => false, + desc => ?DESC("buffer_memory_overload_protection") + })} + ]; +fields(producer_pulsar_message) -> + [ + {key, + mk(string(), #{default => <<"${.clientid}">>, desc => ?DESC("producer_key_template")})}, + {value, mk(string(), #{default => <<"${.}">>, desc => ?DESC("producer_value_template")})} + ]; +fields(producer_resource_opts) -> + SupportedOpts = [ + health_check_interval, + resume_interval, + start_after_created, + start_timeout, + auto_restart_interval + ], + lists:filtermap( + fun + ({health_check_interval = Field, MetaFn}) -> + {true, {Field, override_default(MetaFn, 1_000)}}; + ({Field, _Meta}) -> + lists:member(Field, SupportedOpts) + end, + emqx_resource_schema:fields("creation_opts") + ); +fields(auth_basic) -> + [ + {username, mk(binary(), #{required => true, desc => ?DESC("auth_basic_username")})}, + {password, + mk(binary(), #{ + required => true, + desc => ?DESC("auth_basic_password"), + sensitive => true, + converter => fun emqx_schema:password_converter/2 + })} + ]; +fields(auth_token) -> + [ + {jwt, + mk(binary(), #{ + required => true, + desc => ?DESC("auth_token_jwt"), + sensitive => true, + converter => fun emqx_schema:password_converter/2 + })} + ]; +fields("get_" ++ Type) -> + emqx_bridge_schema:status_fields() ++ fields("post_" ++ Type); +fields("put_" ++ Type) -> + fields("config_" ++ Type); +fields("post_" ++ Type) -> + [type_field(), name_field() | fields("config_" ++ Type)]; +fields("config_producer") -> + fields(pulsar_producer). + +desc(pulsar_producer) -> + ?DESC(pulsar_producer_struct); +desc(producer_resource_opts) -> + ?DESC(emqx_resource_schema, "creation_opts"); +desc("get_" ++ Type) when Type =:= "producer" -> + ["Configuration for Pulsar using `GET` method."]; +desc("put_" ++ Type) when Type =:= "producer" -> + ["Configuration for Pulsar using `PUT` method."]; +desc("post_" ++ Type) when Type =:= "producer" -> + ["Configuration for Pulsar using `POST` method."]; +desc(Name) -> + lists:member(Name, struct_names()) orelse throw({missing_desc, Name}), + ?DESC(Name). + +conn_bridge_examples(_Method) -> + [ + #{ + <<"pulsar_producer">> => #{ + summary => <<"Pulsar Producer Bridge">>, + value => #{todo => true} + } + } + ]. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Name) -> hoconsc:ref(?MODULE, Name). + +type_field() -> + {type, mk(hoconsc:enum([pulsar_producer]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. + +struct_names() -> + [ + auth_basic, + auth_token, + producer_buffer, + producer_pulsar_message + ]. + +override_default(OriginalFn, NewDefault) -> + fun + (default) -> NewDefault; + (Field) -> OriginalFn(Field) + end. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl new file mode 100644 index 000000000..bedf42cf6 --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl @@ -0,0 +1,14 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_pulsar_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_bridge_pulsar_sup:start_link(). + +stop(_State) -> + ok. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl new file mode 100644 index 000000000..4bc390b91 --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -0,0 +1,396 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_pulsar_impl_producer). + +-include("emqx_bridge_pulsar.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_get_status/2, + on_query/3, + on_query_async/4 +]). + +-type pulsar_client_id() :: atom(). +-type state() :: #{ + pulsar_client_id := pulsar_client_id(), + producers := pulsar_producers:producers(), + sync_timeout := infinity | time:time(), + message_template := message_template() +}. +-type buffer_mode() :: memory | disk | hybrid. +-type compression_mode() :: no_compression | snappy | zlib. +-type partition_strategy() :: random | roundrobin | first_key_dispatch. +-type message_template_raw() :: #{ + key := binary(), + value := binary() +}. +-type message_template() :: #{ + key := emqx_plugin_libs_rule:tmpl_token(), + value := emqx_plugin_libs_rule:tmpl_token() +}. +-type config() :: #{ + authentication := _, + batch_size := pos_integer(), + bridge_name := atom(), + buffer := #{ + mode := buffer_mode(), + per_partition_limit := emqx_schema:byte_size(), + segment_bytes := emqx_schema:byte_size(), + memory_overload_protection := boolean() + }, + compression := compression_mode(), + max_batch_bytes := emqx_schema:bytesize(), + message := message_template_raw(), + pulsar_topic := binary(), + retention_period := infinity | emqx_schema:duration_ms(), + send_buffer := emqx_schema:bytesize(), + servers := binary(), + ssl := _, + strategy := partition_strategy(), + sync_timeout := emqx_schema:duration_ms() +}. + +%%------------------------------------------------------------------------------------- +%% `emqx_resource' API +%%------------------------------------------------------------------------------------- + +callback_mode() -> async_if_possible. + +%% there are no queries to be made to this bridge, so we say that +%% buffer is supported so we don't spawn unused resource buffer +%% workers. +is_buffer_supported() -> true. + +-spec on_start(manager_id(), config()) -> {ok, state()}. +on_start(InstanceId, Config) -> + #{ + authentication := _Auth, + bridge_name := BridgeName, + servers := Servers0, + ssl := SSL + } = Config, + Servers = format_servers(Servers0), + ClientId = make_client_id(InstanceId, BridgeName), + SSLOpts = emqx_tls_lib:to_client_opts(SSL), + ClientOpts = #{ + ssl_opts => SSLOpts, + conn_opts => conn_opts(Config) + }, + case pulsar:ensure_supervised_client(ClientId, Servers, ClientOpts) of + {ok, _Pid} -> + ?SLOG(info, #{ + msg => "pulsar_client_started", + instance_id => InstanceId, + pulsar_hosts => Servers + }); + {error, Error} -> + ?SLOG(error, #{ + msg => "failed_to_start_kafka_client", + instance_id => InstanceId, + pulsar_hosts => Servers, + reason => Error + }), + throw(failed_to_start_pulsar_client) + end, + start_producer(Config, InstanceId, ClientId, ClientOpts). + +-spec on_stop(manager_id(), state()) -> ok. +on_stop(_InstanceId, State) -> + #{ + pulsar_client_id := ClientId, + producers := Producers + } = State, + stop_producers(ClientId, Producers), + stop_client(ClientId), + ?tp(pulsar_bridge_stopped, #{instance_id => _InstanceId}), + ok. + +-spec on_get_status(manager_id(), state()) -> connected | disconnected. +on_get_status(_InstanceId, State) -> + #{ + pulsar_client_id := ClientId, + producers := Producers + } = State, + case pulsar_client_sup:find_client(ClientId) of + {ok, Pid} -> + try pulsar_client:get_status(Pid) of + true -> + get_producer_status(Producers); + false -> + disconnected + catch + error:timeout -> + disconnected; + exit:{noproc, _} -> + disconnected + end; + {error, _} -> + disconnected + end. + +-spec on_query(manager_id(), {send_message, map()}, state()) -> ok | {error, timeout}. +on_query(_InstanceId, {send_message, Message}, State) -> + #{ + producers := Producers, + sync_timeout := SyncTimeout, + message_template := MessageTemplate + } = State, + PulsarMessage = render_message(Message, MessageTemplate), + try + pulsar:send_sync(Producers, [PulsarMessage], SyncTimeout) + catch + error:timeout -> + {error, timeout} + end. + +-spec on_query_async( + manager_id(), {send_message, map()}, {ReplyFun :: function(), Args :: list()}, state() +) -> + {ok, pid()}. +on_query_async(_InstanceId, {send_message, Message}, AsyncReplyFn, State) -> + #{ + producers := Producers, + message_template := MessageTemplate + } = State, + PulsarMessage = render_message(Message, MessageTemplate), + pulsar:send(Producers, [PulsarMessage], #{callback_fn => AsyncReplyFn}). + +%%------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------- + +-spec to_bin(atom() | string() | binary()) -> binary(). +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(L) when is_list(L) -> + list_to_binary(L); +to_bin(B) when is_binary(B) -> + B. + +-spec format_servers(binary()) -> [string()]. +format_servers(Servers0) -> + Servers1 = emqx_schema:parse_servers(Servers0, ?PULSAR_HOST_OPTIONS), + lists:map( + fun({Scheme, Host, Port}) -> + Scheme ++ "://" ++ Host ++ ":" ++ integer_to_list(Port) + end, + Servers1 + ). + +-spec make_client_id(manager_id(), atom() | binary()) -> pulsar_client_id(). +make_client_id(InstanceId, BridgeName) -> + case is_dry_run(InstanceId) of + true -> + pulsar_producer_probe; + false -> + ClientIdBin = iolist_to_binary([ + <<"pulsar_producer:">>, + to_bin(BridgeName), + <<":">>, + to_bin(node()) + ]), + binary_to_atom(ClientIdBin) + end. + +-spec is_dry_run(manager_id()) -> boolean(). +is_dry_run(InstanceId) -> + TestIdStart = string:find(InstanceId, ?TEST_ID_PREFIX), + case TestIdStart of + nomatch -> + false; + _ -> + string:equal(TestIdStart, InstanceId) + end. + +conn_opts(#{authentication := none}) -> + #{}; +conn_opts(#{authentication := #{username := Username, password := Password}}) -> + #{ + auth_data => iolist_to_binary([Username, <<":">>, Password]), + auth_method_name => <<"basic">> + }; +conn_opts(#{authentication := #{jwt := JWT}}) -> + #{ + auth_data => JWT, + auth_method_name => <<"token">> + }. + +-spec replayq_dir(pulsar_client_id()) -> string(). +replayq_dir(ClientId) -> + filename:join([emqx:data_dir(), "pulsar", to_bin(ClientId)]). + +-spec producer_name(pulsar_client_id()) -> atom(). +producer_name(ClientId) -> + ClientIdBin = to_bin(ClientId), + binary_to_atom( + iolist_to_binary([ + <<"producer-">>, + ClientIdBin + ]) + ). + +-spec start_producer(config(), manager_id(), pulsar_client_id(), map()) -> {ok, state()}. +start_producer(Config, InstanceId, ClientId, ClientOpts) -> + #{ + conn_opts := ConnOpts, + ssl_opts := SSLOpts + } = ClientOpts, + #{ + batch_size := BatchSize, + buffer := #{ + mode := BufferMode, + per_partition_limit := PerPartitionLimit, + segment_bytes := SegmentBytes, + memory_overload_protection := MemOLP0 + }, + compression := Compression, + max_batch_bytes := MaxBatchBytes, + message := MessageTemplateOpts, + pulsar_topic := PulsarTopic0, + retention_period := RetentionPeriod, + send_buffer := SendBuffer, + strategy := Strategy, + sync_timeout := SyncTimeout + } = Config, + {OffloadMode, ReplayQDir} = + case BufferMode of + memory -> {false, false}; + disk -> {false, replayq_dir(ClientId)}; + hybrid -> {true, replayq_dir(ClientId)} + end, + MemOLP = + case os:type() of + {unix, linux} -> MemOLP0; + _ -> false + end, + ReplayQOpts = #{ + replayq_dir => ReplayQDir, + replayq_offload_mode => OffloadMode, + replayq_max_total_bytes => PerPartitionLimit, + replayq_seg_bytes => SegmentBytes, + drop_if_highmem => MemOLP + }, + ProducerName = producer_name(ClientId), + MessageTemplate = compile_message_template(MessageTemplateOpts), + ProducerOpts0 = + #{ + batch_size => BatchSize, + compression => Compression, + conn_opts => ConnOpts, + max_batch_bytes => MaxBatchBytes, + name => ProducerName, + retention_period => RetentionPeriod, + ssl_opts => SSLOpts, + strategy => Strategy, + tcp_opts => [{sndbuf, SendBuffer}] + }, + ProducerOpts = maps:merge(ReplayQOpts, ProducerOpts0), + PulsarTopic = binary_to_list(PulsarTopic0), + try pulsar:ensure_supervised_producers(ClientId, PulsarTopic, ProducerOpts) of + {ok, Producers} -> + State = #{ + pulsar_client_id => ClientId, + producers => Producers, + sync_timeout => SyncTimeout, + message_template => MessageTemplate + }, + ?tp(pulsar_producer_bridge_started, #{}), + {ok, State} + catch + Kind:Error:Stacktrace -> + ?SLOG(error, #{ + msg => "failed_to_start_pulsar_producer", + instance_id => InstanceId, + kind => Kind, + reason => Error, + stacktrace => Stacktrace + }), + stop_client(ClientId), + throw(failed_to_start_pulsar_producer) + end. + +-spec stop_client(pulsar_client_id()) -> ok. +stop_client(ClientId) -> + _ = log_when_error( + fun() -> + ok = pulsar:stop_and_delete_supervised_client(ClientId), + ?tp(pulsar_bridge_client_stopped, #{pulsar_client_id => ClientId}), + ok + end, + #{ + msg => "failed_to_delete_pulsar_client", + pulsar_client_id => ClientId + } + ), + ok. + +-spec stop_producers(pulsar_client_id(), pulsar_producers:producers()) -> ok. +stop_producers(ClientId, Producers) -> + _ = log_when_error( + fun() -> + ok = pulsar:stop_and_delete_supervised_producers(Producers), + ?tp(pulsar_bridge_producer_stopped, #{pulsar_client_id => ClientId}), + ok + end, + #{ + msg => "failed_to_delete_pulsar_producer", + pulsar_client_id => ClientId + } + ), + ok. + +log_when_error(Fun, Log) -> + try + Fun() + catch + C:E -> + ?SLOG(error, Log#{ + exception => C, + reason => E + }) + end. + +-spec compile_message_template(message_template_raw()) -> message_template(). +compile_message_template(TemplateOpts) -> + KeyTemplate = maps:get(key, TemplateOpts, <<"${.clientid}">>), + ValueTemplate = maps:get(value, TemplateOpts, <<"${.}">>), + #{ + key => preproc_tmpl(KeyTemplate), + value => preproc_tmpl(ValueTemplate) + }. + +preproc_tmpl(Template) -> + emqx_plugin_libs_rule:preproc_tmpl(Template). + +render_message( + Message, #{key := KeyTemplate, value := ValueTemplate} +) -> + #{ + key => render(Message, KeyTemplate), + value => render(Message, ValueTemplate) + }. + +render(Message, Template) -> + Opts = #{ + var_trans => fun + (undefined) -> <<"">>; + (X) -> emqx_plugin_libs_rule:bin(X) + end, + return => full_binary + }, + emqx_plugin_libs_rule:proc_tmpl(Template, Message, Opts). + +get_producer_status(Producers) -> + case pulsar_producers:all_connected(Producers) of + true -> connected; + false -> connecting + end. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl new file mode 100644 index 000000000..17121beab --- /dev/null +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_pulsar_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 0, + period => 1 + }, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl new file mode 100644 index 000000000..f86dbc65d --- /dev/null +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -0,0 +1,819 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_pulsar_impl_producer_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(BRIDGE_TYPE_BIN, <<"pulsar_producer">>). +-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_pulsar]). +-define(RULE_TOPIC, "mqtt/rule"). +-define(RULE_TOPIC_BIN, <>). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, plain}, + {group, tls} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + OnlyOnceTCs = only_once_tests(), + TCs = AllTCs -- OnlyOnceTCs, + [ + {plain, AllTCs}, + {tls, TCs} + ]. + +only_once_tests() -> + [t_create_via_http]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), + _ = application:stop(emqx_connector), + ok. + +init_per_group(plain = Type, Config) -> + PulsarHost = os:getenv("PULSAR_PLAIN_HOST", "toxiproxy"), + PulsarPort = list_to_integer(os:getenv("PULSAR_PLAIN_PORT", "6652")), + ProxyName = "pulsar_plain", + case emqx_common_test_helpers:is_tcp_server_available(PulsarHost, PulsarPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {pulsar_host, PulsarHost}, + {pulsar_port, PulsarPort}, + {pulsar_type, Type}, + {use_tls, false} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_pulsar); + _ -> + {skip, no_pulsar} + end + end; +init_per_group(tls = Type, Config) -> + PulsarHost = os:getenv("PULSAR_TLS_HOST", "toxiproxy"), + PulsarPort = list_to_integer(os:getenv("PULSAR_TLS_PORT", "6653")), + ProxyName = "pulsar_tls", + case emqx_common_test_helpers:is_tcp_server_available(PulsarHost, PulsarPort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {pulsar_host, PulsarHost}, + {pulsar_port, PulsarPort}, + {pulsar_type, Type}, + {use_tls, true} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_pulsar); + _ -> + {skip, no_pulsar} + end + end; +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when + Group =:= plain +-> + common_end_per_group(Config), + ok; +end_per_group(_Group, _Config) -> + ok. + +common_init_per_group() -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + application:load(emqx_bridge), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps(?APPS), + {ok, _} = application:ensure_all_started(emqx_connector), + emqx_mgmt_api_test_util:init_suite(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + MQTTTopic = <<"mqtt/topic/", UniqueNum/binary>>, + [ + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {mqtt_topic, MQTTTopic} + ]. + +common_end_per_group(Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + ok. + +init_per_testcase(TestCase, Config) -> + common_init_per_testcase(TestCase, Config). + +end_per_testcase(_Testcase, Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + stop_consumer(Config), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), + ok = snabbkaffe:stop(), + ok + end. + +common_init_per_testcase(TestCase, Config0) -> + ct:timetrap(timer:seconds(60)), + delete_all_bridges(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + PulsarTopic = + << + (atom_to_binary(TestCase))/binary, + UniqueNum/binary + >>, + PulsarType = ?config(pulsar_type, Config0), + Config1 = [{pulsar_topic, PulsarTopic} | Config0], + {Name, ConfigString, PulsarConfig} = pulsar_config( + TestCase, PulsarType, Config1 + ), + ConsumerConfig = start_consumer(TestCase, Config1), + Config = ConsumerConfig ++ Config1, + ok = snabbkaffe:start_trace(), + [ + {pulsar_name, Name}, + {pulsar_config_string, ConfigString}, + {pulsar_config, PulsarConfig} + | Config + ]. + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +pulsar_config(TestCase, _PulsarType, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + PulsarHost = ?config(pulsar_host, Config), + PulsarPort = ?config(pulsar_port, Config), + PulsarTopic = ?config(pulsar_topic, Config), + AuthType = proplists:get_value(sasl_auth_mechanism, Config, none), + UseTLS = proplists:get_value(use_tls, Config, false), + Name = << + (atom_to_binary(TestCase))/binary, UniqueNum/binary + >>, + MQTTTopic = proplists:get_value(mqtt_topic, Config, <<"mqtt/topic/", UniqueNum/binary>>), + Prefix = + case UseTLS of + true -> <<"pulsar+ssl://">>; + false -> <<"pulsar://">> + end, + ServerURL = iolist_to_binary([ + Prefix, + PulsarHost, + ":", + integer_to_binary(PulsarPort) + ]), + ConfigString = + io_lib:format( + "bridges.pulsar_producer.~s {\n" + " enable = true\n" + " servers = \"~s\"\n" + " sync_timeout = 5s\n" + " compression = no_compression\n" + " send_buffer = 1MB\n" + " retention_period = infinity\n" + " max_batch_bytes = 900KB\n" + " batch_size = 1\n" + " strategy = random\n" + " buffer {\n" + " mode = memory\n" + " per_partition_limit = 10MB\n" + " segment_bytes = 5MB\n" + " memory_overload_protection = true\n" + " }\n" + " message {\n" + " key = \"${.clientid}\"\n" + " value = \"${.}\"\n" + " }\n" + "~s" + " ssl {\n" + " enable = ~p\n" + " verify = verify_none\n" + " server_name_indication = \"auto\"\n" + " }\n" + " pulsar_topic = \"~s\"\n" + " local_topic = \"~s\"\n" + "}\n", + [ + Name, + ServerURL, + authentication(AuthType), + UseTLS, + PulsarTopic, + MQTTTopic + ] + ), + {Name, ConfigString, parse_and_check(ConfigString, Name)}. + +parse_and_check(ConfigString, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + TypeBin = ?BRIDGE_TYPE_BIN, + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf, + Config. + +authentication(_) -> + " authentication = none\n". + +resource_id(Config) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(pulsar_name, Config), + emqx_bridge_resource:resource_id(Type, Name). + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(pulsar_name, Config), + PulsarConfig0 = ?config(pulsar_config, Config), + PulsarConfig = emqx_utils_maps:deep_merge(PulsarConfig0, Overrides), + emqx_bridge:create(Type, Name, PulsarConfig). + +create_bridge_api(Config) -> + create_bridge_api(Config, _Overrides = #{}). + +create_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(pulsar_name, Config), + PulsarConfig0 = ?config(pulsar_config, Config), + PulsarConfig = emqx_utils_maps:deep_merge(PulsarConfig0, Overrides), + Params = PulsarConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("creating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; + Error -> + Error + end, + ct:pal("bridge create result: ~p", [Res]), + Res. + +update_bridge_api(Config) -> + update_bridge_api(Config, _Overrides = #{}). + +update_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(pulsar_name, Config), + PulsarConfig0 = ?config(pulsar_config, Config), + PulsarConfig = emqx_utils_maps:deep_merge(PulsarConfig0, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), + Params = PulsarConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("updating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; + Error -> Error + end, + ct:pal("bridge update result: ~p", [Res]), + Res. + +probe_bridge_api(Config) -> + probe_bridge_api(Config, _Overrides = #{}). + +probe_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(pulsar_name, Config), + PulsarConfig = ?config(pulsar_config, Config), + Params0 = PulsarConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Params = maps:merge(Params0, Overrides), + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("probing bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; + Error -> Error + end, + ct:pal("bridge probe result: ~p", [Res]), + Res. + +start_consumer(TestCase, Config) -> + PulsarHost = ?config(pulsar_host, Config), + PulsarPort = ?config(pulsar_port, Config), + PulsarTopic = ?config(pulsar_topic, Config), + UseTLS = ?config(use_tls, Config), + %% FIXME: patch pulsar to accept binary urls... + Scheme = + case UseTLS of + true -> <<"pulsar+ssl://">>; + false -> <<"pulsar://">> + end, + URL = + binary_to_list( + <> + ), + ConnOpts = #{}, + ConsumerClientId = TestCase, + CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"), + SSLOpts = #{ + enable => UseTLS, + keyfile => filename:join([CertsPath, "key.pem"]), + certfile => filename:join([CertsPath, "cert.pem"]), + cacertfile => filename:join([CertsPath, "cacert.pem"]) + }, + {ok, _ClientPid} = pulsar:ensure_supervised_client( + ConsumerClientId, + [URL], + #{ + conn_opts => ConnOpts, + ssl_opts => emqx_tls_lib:to_client_opts(SSLOpts) + } + ), + ConsumerOpts = #{ + cb_init_args => #{send_to => self()}, + cb_module => pulsar_echo_consumer, + sub_type => 'Shared', + subscription => atom_to_list(TestCase), + max_consumer_num => 1, + %% Note! This must not coincide with the client + %% id, or else weird bugs will happen, like the + %% consumer never starts... + name => test_consumer, + consumer_id => 1, + conn_opts => ConnOpts + }, + {ok, Consumer} = pulsar:ensure_supervised_consumers( + ConsumerClientId, + PulsarTopic, + ConsumerOpts + ), + %% since connection is async, and there's currently no way to + %% specify the subscription initial position as `Earliest', we + %% need to wait until the consumer is connected to avoid + %% flakiness. + ok = wait_until_consumer_connected(Consumer), + [ + {consumer_client_id, ConsumerClientId}, + {pulsar_consumer, Consumer} + ]. + +stop_consumer(Config) -> + ConsumerClientId = ?config(consumer_client_id, Config), + Consumer = ?config(pulsar_consumer, Config), + ok = pulsar:stop_and_delete_supervised_consumers(Consumer), + ok = pulsar:stop_and_delete_supervised_client(ConsumerClientId), + ok. + +wait_until_consumer_connected(Consumer) -> + ?retry( + _Sleep = 300, + _Attempts0 = 20, + true = pulsar_consumers:all_connected(Consumer) + ), + ok. + +wait_until_producer_connected() -> + wait_until_connected(pulsar_producers_sup, pulsar_producer). + +wait_until_connected(SupMod, Mod) -> + Pids = [ + P + || {_Name, SupPid, _Type, _Mods} <- supervisor:which_children(SupMod), + P <- element(2, process_info(SupPid, links)), + case proc_lib:initial_call(P) of + {Mod, init, _} -> true; + _ -> false + end + ], + ?retry( + _Sleep = 300, + _Attempts0 = 20, + lists:foreach(fun(P) -> {connected, _} = sys:get_state(P) end, Pids) + ), + ok. + +create_rule_and_action_http(Config) -> + PulsarName = ?config(pulsar_name, Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, PulsarName), + Params = #{ + enable => true, + sql => <<"SELECT * FROM \"", ?RULE_TOPIC, "\"">>, + actions => [BridgeId] + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ct:pal("rule action params: ~p", [Params]), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +receive_consumed(Timeout) -> + receive + {pulsar_message, #{payloads := Payloads}} -> + lists:map(fun try_decode_json/1, Payloads) + after Timeout -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + ct:fail("no message consumed") + end. + +try_decode_json(Payload) -> + case emqx_utils_json:safe_decode(Payload, [return_maps]) of + {error, _} -> + Payload; + {ok, JSON} -> + JSON + end. + +cluster(Config) -> + PrivDataDir = ?config(priv_dir, Config), + PeerModule = + case os:getenv("IS_CI") of + false -> + slave; + _ -> + ct_slave + end, + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_pulsar]}, + {listener_ports, []}, + {peer_mod, PeerModule}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {start_autocluster, true}, + {schema_mod, emqx_ee_conf_schema}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, [broker, router]), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster: ~p", [Cluster]), + Cluster. + +start_cluster(Cluster) -> + Nodes = + [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + on_exit(fun() -> + emqx_utils:pmap( + fun(N) -> + ct:pal("stopping ~p", [N]), + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + Nodes. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_start_and_produce_ok(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + ResourceId = resource_id(Config), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + QoS = 0, + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + ?check_trace( + begin + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + %% Publish using local topic. + Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), + emqx:publish(Message0), + %% Publish using rule engine. + Message1 = emqx_message:make(ClientId, QoS, ?RULE_TOPIC_BIN, Payload), + emqx:publish(Message1), + + #{rule_id => RuleId} + end, + fun(#{rule_id := RuleId}, _Trace) -> + Data0 = receive_consumed(5_000), + ?assertMatch( + [ + #{ + <<"clientid">> := ClientId, + <<"event">> := <<"message.publish">>, + <<"payload">> := Payload, + <<"topic">> := MQTTTopic + } + ], + Data0 + ), + Data1 = receive_consumed(5_000), + ?assertMatch( + [ + #{ + <<"clientid">> := ClientId, + <<"event">> := <<"message.publish">>, + <<"payload">> := Payload, + <<"topic">> := ?RULE_TOPIC_BIN + } + ], + Data1 + ), + ?retry( + _Sleep = 100, + _Attempts0 = 20, + begin + ?assertMatch( + #{ + counters := #{ + dropped := 0, + failed := 0, + late_reply := 0, + matched := 2, + received := 0, + retried := 0, + success := 2 + } + }, + emqx_resource_manager:get_metrics(ResourceId) + ), + ?assertEqual( + 1, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.success') + ), + ?assertEqual( + 0, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.failed') + ), + ok + end + ), + ok + end + ), + ok. + +%% Under normal operations, the bridge will be called async via +%% `simple_async_query'. +t_sync_query(Config) -> + ResourceId = resource_id(Config), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + Message = {send_message, #{payload => Payload}}, + ?assertMatch( + {ok, #{sequence_id := _}}, emqx_resource:simple_sync_query(ResourceId, Message) + ), + ok + end, + [] + ), + ok. + +t_create_via_http(Config) -> + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + + %% lightweight matrix testing some configs + ?assertMatch( + {ok, _}, + update_bridge_api( + Config, + #{ + <<"buffer">> => + #{<<"mode">> => <<"disk">>} + } + ) + ), + ?assertMatch( + {ok, _}, + update_bridge_api( + Config, + #{ + <<"buffer">> => + #{ + <<"mode">> => <<"hybrid">>, + <<"memory_overload_protection">> => true + } + } + ) + ), + ok + end, + [] + ), + ok. + +t_start_stop(Config) -> + PulsarName = ?config(pulsar_name, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + %% Check that the bridge probe API doesn't leak atoms. + redbug:start( + [ + "emqx_resource_manager:health_check_interval -> return", + "emqx_resource_manager:with_health_check -> return" + ], + [{msgs, 100}, {time, 30_000}] + ), + ProbeRes0 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), + AtomsBefore = erlang:system_info(atom_count), + %% Probe again; shouldn't have created more atoms. + ProbeRes1 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes1), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), + + %% Now stop the bridge. + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge:disable_enable(disable, ?BRIDGE_TYPE_BIN, PulsarName), + #{?snk_kind := pulsar_bridge_stopped}, + 5_000 + ) + ), + + ok + end, + fun(Trace) -> + %% one for each probe, one for real + ?assertMatch([_, _, _], ?of_kind(pulsar_bridge_producer_stopped, Trace)), + ?assertMatch([_, _, _], ?of_kind(pulsar_bridge_client_stopped, Trace)), + ?assertMatch([_, _, _], ?of_kind(pulsar_bridge_stopped, Trace)), + ok + end + ), + ok. + +t_on_get_status(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ResourceId = resource_id(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ct:sleep(500), + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)) + end), + %% Check that it recovers itself. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + ok. + +t_cluster(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + ResourceId = resource_id(Config), + Cluster = cluster(Config), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + QoS = 0, + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + ?check_trace( + begin + Nodes = [N1, N2 | _] = start_cluster(Cluster), + {ok, SRef0} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := pulsar_producer_bridge_started}), + length(Nodes), + 15_000 + ), + {ok, _} = erpc:call(N1, fun() -> create_bridge(Config) end), + {ok, _} = snabbkaffe:receive_events(SRef0), + lists:foreach( + fun(N) -> + ?retry( + _Sleep = 1_000, + _Attempts0 = 20, + ?assertEqual( + {ok, connected}, + erpc:call(N, emqx_resource_manager, health_check, [ResourceId]), + #{node => N} + ) + ) + end, + Nodes + ), + erpc:multicall(Nodes, fun wait_until_producer_connected/0), + Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), + erpc:call(N2, emqx, publish, [Message0]), + + lists:foreach( + fun(N) -> + ?assertEqual( + {ok, connected}, + erpc:call(N, emqx_resource_manager, health_check, [ResourceId]), + #{node => N} + ) + end, + Nodes + ), + + ok + end, + fun(_Trace) -> + Data0 = receive_consumed(10_000), + ?assertMatch( + [ + #{ + <<"clientid">> := ClientId, + <<"event">> := <<"message.publish">>, + <<"payload">> := Payload, + <<"topic">> := MQTTTopic + } + ], + Data0 + ), + ok + end + ), + ok. diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE_data/pulsar_echo_consumer.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE_data/pulsar_echo_consumer.erl new file mode 100644 index 000000000..834978851 --- /dev/null +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE_data/pulsar_echo_consumer.erl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(pulsar_echo_consumer). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% pulsar consumer API +-export([init/2, handle_message/3]). + +init(Topic, Args) -> + ct:pal("consumer init: ~p", [#{topic => Topic, args => Args}]), + SendTo = maps:get(send_to, Args), + ?tp(pulsar_echo_consumer_init, #{topic => Topic}), + {ok, #{topic => Topic, send_to => SendTo}}. + +handle_message(Message, Payloads, State) -> + #{send_to := SendTo, topic := Topic} = State, + ct:pal( + "pulsar consumer received:\n ~p", + [#{message => Message, payloads => Payloads}] + ), + SendTo ! {pulsar_message, #{topic => Topic, message => Message, payloads => Payloads}}, + ?tp(pulsar_echo_consumer_message, #{topic => Topic, message => Message, payloads => Payloads}), + {ok, 'Individual', State}. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 877b35fff..f6a0ebebf 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -165,8 +165,13 @@ create(MgrId, ResId, Group, ResourceType, Config, Opts) -> create_dry_run(ResourceType, Config) -> ResId = make_test_id(), MgrId = set_new_owner(ResId), + Opts = + case is_map(Config) of + true -> maps:get(resource_opts, Config, #{}); + false -> #{} + end, ok = emqx_resource_manager_sup:ensure_child( - MgrId, ResId, <<"dry_run">>, ResourceType, Config, #{} + MgrId, ResId, <<"dry_run">>, ResourceType, Config, Opts ), case wait_for_ready(ResId, 5000) of ok -> diff --git a/changes/ee/feat-10378.en.md b/changes/ee/feat-10378.en.md new file mode 100644 index 000000000..ebdd299c8 --- /dev/null +++ b/changes/ee/feat-10378.en.md @@ -0,0 +1 @@ +Implement Pulsar Producer Bridge, which supports publishing messages to Pulsar from MQTT topics. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 7dc8882b3..5544825f8 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -9,7 +9,9 @@ telemetry, emqx_bridge_kafka, emqx_bridge_gcp_pubsub, - emqx_bridge_opents + emqx_bridge_cassandra, + emqx_bridge_opents, + emqx_bridge_pulsar ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 4b83fda3f..38f471ca2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -36,7 +36,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), ref(emqx_ee_bridge_sqlserver, Method), - ref(emqx_bridge_opents, Method) + ref(emqx_bridge_opents, Method), + ref(emqx_bridge_pulsar, Method ++ "_producer") ]. schema_modules() -> @@ -57,7 +58,8 @@ schema_modules() -> emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, emqx_ee_bridge_sqlserver, - emqx_bridge_opents + emqx_bridge_opents, + emqx_bridge_pulsar ]. examples(Method) -> @@ -97,7 +99,8 @@ resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; resource_type(sqlserver) -> emqx_ee_connector_sqlserver; -resource_type(opents) -> emqx_bridge_opents_connector. +resource_type(opents) -> emqx_bridge_opents_connector; +resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer. fields(bridges) -> [ @@ -165,7 +168,8 @@ fields(bridges) -> required => false } )} - ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ + ] ++ kafka_structs() ++ pulsar_structs() ++ mongodb_structs() ++ influxdb_structs() ++ + redis_structs() ++ pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs(). mongodb_structs() -> @@ -202,6 +206,18 @@ kafka_structs() -> )} ]. +pulsar_structs() -> + [ + {pulsar_producer, + mk( + hoconsc:map(name, ref(emqx_bridge_pulsar, pulsar_producer)), + #{ + desc => <<"Pulsar Producer Bridge Config">>, + required => false + } + )} + ]. + influxdb_structs() -> [ {Protocol, diff --git a/mix.exs b/mix.exs index e2230d55d..97fdd732a 100644 --- a/mix.exs +++ b/mix.exs @@ -169,7 +169,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_redis, :emqx_bridge_rocketmq, :emqx_bridge_tdengine, - :emqx_bridge_timescale + :emqx_bridge_timescale, + :emqx_bridge_pulsar ]) end @@ -360,6 +361,7 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, + emqx_bridge_pulsar: :permanent, emqx_bridge_gcp_pubsub: :permanent, emqx_bridge_cassandra: :permanent, emqx_bridge_opents: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 3c863046f..020285a44 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -454,6 +454,7 @@ relx_apps_per_edition(ee) -> emqx_ee_connector, emqx_ee_bridge, emqx_bridge_kafka, + emqx_bridge_pulsar, emqx_bridge_gcp_pubsub, emqx_bridge_cassandra, emqx_bridge_opents, diff --git a/rel/i18n/emqx_bridge_pulsar.hocon b/rel/i18n/emqx_bridge_pulsar.hocon new file mode 100644 index 000000000..1d2bb4599 --- /dev/null +++ b/rel/i18n/emqx_bridge_pulsar.hocon @@ -0,0 +1,176 @@ +emqx_bridge_pulsar { + auth_basic { + desc = "Parameters for basic authentication." + label = "Basic auth params" + } + + auth_basic_password { + desc = "Basic authentication password." + label = "Password" + } + + auth_basic_username { + desc = "Basic authentication username." + label = "Username" + } + + auth_token { + desc = "Parameters for token authentication." + label = "Token auth params" + } + + auth_token_jwt { + desc = "JWT authentication token." + label = "JWT" + } + + authentication { + desc = "Authentication configs." + label = "Authentication" + } + + buffer_memory_overload_protection { + desc = "Applicable when buffer mode is set to memory\n" + "EMQX will drop old buffered messages under high memory pressure." + " The high memory threshold is defined in config sysmon.os.sysmem_high_watermark." + " NOTE: This config only works on Linux." + label = "Memory Overload Protection" + } + + buffer_mode { + desc = "Message buffer mode.\n" + "\n" + "memory: Buffer all messages in memory. The messages will be lost" + " in case of EMQX node restart\ndisk: Buffer all messages on disk." + " The messages on disk are able to survive EMQX node restart.\n" + "hybrid: Buffer message in memory first, when up to certain limit" + " (see segment_bytes config for more information), then start offloading" + " messages to disk, Like memory mode, the messages will be lost in" + " case of EMQX node restart." + label = "Buffer Mode" + } + + buffer_per_partition_limit { + desc = "Number of bytes allowed to buffer for each Pulsar partition." + " When this limit is exceeded, old messages will be dropped in a trade for credits" + " for new messages to be buffered." + label = "Per-partition Buffer Limit" + } + + buffer_segment_bytes { + desc = "Applicable when buffer mode is set to disk or hybrid.\n" + "This value is to specify the size of each on-disk buffer file." + label = "Segment File Bytes" + } + + config_enable { + desc = "Enable (true) or disable (false) this Pulsar bridge." + label = "Enable or Disable" + } + + desc_name { + desc = "Bridge name, used as a human-readable description of the bridge." + label = "Bridge Name" + } + + desc_type { + desc = "The Bridge Type" + label = "Bridge Type" + } + + producer_batch_size { + desc = "Maximum number of individual requests to batch in a Pulsar message." + label = "Batch size" + } + + producer_buffer { + desc = "Configure producer message buffer.\n\n" + "Tell Pulsar producer how to buffer messages when EMQX has more messages to" + " send than Pulsar can keep up, or when Pulsar is down." + label = "Message Buffer" + } + + producer_compression { + desc = "Compression method." + label = "Compression" + } + + producer_key_template { + desc = "Template to render Pulsar message key." + label = "Message Key" + } + + producer_local_topic { + desc = "MQTT topic or topic filter as data source (bridge input)." + " If rule action is used as data source, this config should be left empty," + " otherwise messages will be duplicated in Pulsar." + label = "Source MQTT Topic" + } + + producer_max_batch_bytes { + desc = "Maximum bytes to collect in a Pulsar message batch. Most of the Pulsar brokers" + " default to a limit of 5 MB batch size. EMQX's default value is less than 5 MB in" + " order to compensate Pulsar message encoding overheads (especially when each individual" + " message is very small). When a single message is over the limit, it is still" + " sent (as a single element batch)." + label = "Max Batch Bytes" + } + + producer_message_opts { + desc = "Template to render a Pulsar message." + label = "Pulsar Message Template" + } + + producer_pulsar_message { + desc = "Template to render a Pulsar message." + label = "Pulsar Message Template" + } + + producer_pulsar_topic { + desc = "Pulsar topic name" + label = "Pulsar topic name" + } + + producer_retention_period { + desc = "The amount of time messages will be buffered while there is no connection to" + " the Pulsar broker. Longer times mean that more memory/disk will be used" + label = "Retention Period" + } + + producer_send_buffer { + desc = "Fine tune the socket send buffer. The default value is tuned for high throughput." + label = "Socket Send Buffer Size" + } + + producer_strategy { + desc = "Partition strategy is to tell the producer how to dispatch messages to Pulsar partitions.\n" + "\n" + "random: Randomly pick a partition for each message.\n" + "roundrobin: Pick each available producer in turn for each message.\n" + "first_key_dispatch: Hash Pulsar message key of the first message in a batch" + " to a partition number." + label = "Partition Strategy" + } + + producer_sync_timeout { + desc = "Maximum wait time for receiving a receipt from Pulsar when publishing synchronously." + label = "Sync publish timeout" + } + + producer_value_template { + desc = "Template to render Pulsar message value." + label = "Message Value" + } + + pulsar_producer_struct { + desc = "Configuration for a Pulsar bridge." + label = "Pulsar Bridge Configuration" + } + + servers { + desc = "A comma separated list of Pulsar URLs in the form scheme://host[:port]" + " for the client to connect to. The supported schemes are pulsar:// (default)" + " and pulsar+ssl://. The default port is 6650." + label = "Servers" + } +} diff --git a/rel/i18n/zh/emqx_bridge_pulsar.hocon b/rel/i18n/zh/emqx_bridge_pulsar.hocon new file mode 100644 index 000000000..3af8652a4 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_pulsar.hocon @@ -0,0 +1,173 @@ +emqx_bridge_pulsar { + + pulsar_producer_struct { + desc = "Pulsar 桥接配置" + label = "Pulsar 桥接配置" + } + + desc_type { + desc = "桥接类型" + label = "桥接类型" + } + + desc_name { + desc = "桥接名字，可读描述" + label = "桥接名字" + } + + config_enable { + desc = "启用（true）或停用该（false）Pulsar 数据桥接。" + label = "启用或停用" + } + + servers { + desc = "以scheme://host[:port]形式分隔的Pulsar URL列表，" + "供客户端连接使用。支持的方案是 pulsar:// （默认）" + "和pulsar+ssl://。默认的端口是6650。" + label = "服务员" + } + + authentication { + desc = "认证参数。" + label = "认证" + } + + producer_batch_size { + desc = "在一个Pulsar消息中批处理的单个请求的最大数量。" + label = "批量大小" + } + + producer_compression { + desc = "压缩方法。" + label = "压缩" + } + + producer_send_buffer { + desc = "TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。" + label = "Socket 发送缓存大小" + } + + producer_sync_timeout { + desc = "同步发布时，从Pulsar接收发送回执的最长等待时间。" + label = "同步发布超时" + } + + auth_basic_username { + desc = "基本认证用户名。" + label = "用户名" + } + + auth_basic_password { + desc = "基本认证密码。" + label = "密码" + } + + auth_token_jwt { + desc = "JWT认证令牌。" + label = "JWT" + } + + producer_max_batch_bytes { + desc = "最大消息批量字节数。" + "大多数 Pulsar 环境的默认最低值是 5 MB，EMQX 的默认值比 5 MB 更小是因为需要" + "补偿 Pulsar 消息编码所需要的额外字节（尤其是当每条消息都很小的情况下）。" + "当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。" + label = "最大批量字节数" + } + + producer_retention_period { + desc = "当没有连接到Pulsar代理时，信息将被缓冲的时间。 较长的时间意味着将使用更多的内存/磁盘" + label = "保留期" + } + + producer_local_topic { + desc = "MQTT 主题数据源由桥接指定，或留空由规则动作指定。" + label = "源 MQTT 主题" + } + + producer_pulsar_topic { + desc = "Pulsar 主题名称" + label = "Pulsar 主题名称" + } + + producer_strategy { + desc = "设置消息发布时应该如何选择 Pulsar 分区。\n\n" + "random: 为每个消息随机选择一个分区。\n" + "roundrobin: 依次为每条信息挑选可用的生产商。\n" + "first_key_dispatch: 将一批信息中的第一条信息的Pulsar信息密钥哈希到一个分区编号。" + label = "分区选择策略" + } + + producer_buffer { + desc = "配置消息缓存的相关参数。\n\n" + "当 EMQX 需要发送的消息超过 Pulsar 处理能力，或者当 Pulsar 临时下线时，EMQX 内部会将消息缓存起来。" + label = "消息缓存" + } + + buffer_mode { + desc = "消息缓存模式。\n" + "memory: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。\n" + "disk: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。\n" + "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 segment_bytes 描述了该限制）后，后续的消息会缓存到磁盘上。" + "与 memory 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。" + label = "缓存模式" + } + + buffer_per_partition_limit { + desc = "为每个 Pulsar 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，" + "为新的消息腾出空间。" + label = "Pulsar 分区缓存上限" + } + + buffer_segment_bytes { + desc = "当缓存模式是 disk 或 hybrid 时适用。" + "该配置用于指定缓存到磁盘上的文件的大小。" + label = "缓存文件大小" + } + + buffer_memory_overload_protection { + desc = "缓存模式是 memory 或 hybrid 时适用。" + "当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。" + "内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。" + "注意，该配置仅在 Linux 系统中有效。" + label = "内存过载保护" + } + + producer_message_opts { + desc = "用于生成 Pulsar 消息的模版。" + label = "Pulsar 消息模版" + } + + producer_key_template { + desc = "生成 Pulsar 消息 Key 的模版。" + label = "消息的 Key" + } + + producer_value_template { + desc = "生成 Pulsar 消息 Value 的模版。" + label = "消息的 Value" + } + + auth_basic { + desc = "基本认证的参数。" + label = "基本认证参数" + } + + auth_token { + desc = "令牌认证的参数。" + label = "Token auth params" + } + + producer_buffer { + desc = "配置消息缓存的相关参数。\n\n" + "当 EMQX 需要发送的消息超过 Pulsar 处理能力，或者当 Pulsar 临时下线时，EMQX 内部会将消息缓存起来。" + label = "消息缓存" + } + + producer_pulsar_message { + desc = "用于生成 Pulsar 消息的模版。" + label = "Pulsar 消息模版" + } + +} diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index c153669f4..307063e84 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -190,7 +190,10 @@ for dep in ${CT_DEPS}; do ;; opents) FILES+=( '.ci/docker-compose-file/docker-compose-opents.yaml' ) - ;; + ;; + pulsar) + FILES+=( '.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From d06ce9c79da949da9a794d7f6413a20be2810dcc Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 18 Apr 2023 12:58:48 -0300 Subject: [PATCH 056/194] docs(license): change license contents after review https://github.com/emqx/emqx/pull/10378#discussion_r1170143535 --- LICENSE | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/LICENSE b/LICENSE index 68bb18ce3..8ff0a9060 100644 --- a/LICENSE +++ b/LICENSE @@ -1,12 +1,7 @@ Source code in this repository is variously licensed under below licenses. -For EMQX: Apache License 2.0, see APL.txt, -which applies to all source files except for lib-ee sub-directory. +For Default: Apache License 2.0, see APL.txt, +which applies to all source files except for folders applied with Business Source License. For EMQX Enterprise (since version 5.0): Business Source License 1.1, -see lib-ee/BSL.txt, which applies to source code in lib-ee -sub-directory and some of the apps under the apps directory. - -Source code under apps that uses BSL License: -- apps/emqx_bridge_kafka -- apps/emqx_bridge_pulsar +see apps/emqx_bridge_kafka/BSL.txt as an example, please check license files under sub directory of apps. From f4a3affd6fd9f98cb0aec107da9d76d0efbc4e08 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 18 Apr 2023 13:00:35 -0300 Subject: [PATCH 057/194] docs: change phrasing after review --- apps/emqx_bridge_kafka/README.md | 2 +- apps/emqx_bridge_pulsar/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md index 80978ff10..f1b0d1f9a 100644 --- a/apps/emqx_bridge_kafka/README.md +++ b/apps/emqx_bridge_kafka/README.md @@ -27,4 +27,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md). # License -See [BSL](./BSL.txt). +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_pulsar/README.md b/apps/emqx_bridge_pulsar/README.md index 09e17d8bb..fbd8bf81d 100644 --- a/apps/emqx_bridge_pulsar/README.md +++ b/apps/emqx_bridge_pulsar/README.md @@ -27,4 +27,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md). # License -See [BSL](./BSL.txt). +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). From 180b6acd9e8d12076be8b1d9424150248db01876 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 18 Apr 2023 13:03:06 -0300 Subject: [PATCH 058/194] docs: remove auto-generated comment --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl index 17121beab..ad7e2ae6f 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl @@ -14,15 +14,6 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). -%% sup_flags() = #{strategy => strategy(), % optional -%% intensity => non_neg_integer(), % optional -%% period => pos_integer()} % optional -%% child_spec() = #{id => child_id(), % mandatory -%% start => mfargs(), % mandatory -%% restart => restart(), % optional -%% shutdown => shutdown(), % optional -%% type => worker(), % optional -%% modules => modules()} % optional init([]) -> SupFlags = #{ strategy => one_for_all, From 1e8dd70a11f68505a01f6f61df7906a87429657c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 09:22:25 -0300 Subject: [PATCH 059/194] chore: fix error message and rename variable --- .../src/emqx_bridge_pulsar_impl_producer.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 4bc390b91..72363389e 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -92,12 +92,12 @@ on_start(InstanceId, Config) -> instance_id => InstanceId, pulsar_hosts => Servers }); - {error, Error} -> + {error, Reason} -> ?SLOG(error, #{ - msg => "failed_to_start_kafka_client", + msg => "failed_to_start_pulsar_client", instance_id => InstanceId, pulsar_hosts => Servers, - reason => Error + reason => Reason }), throw(failed_to_start_pulsar_client) end, From 120d3e70ea0f38cf92b92ba7e603665c7ee9a546 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 09:28:58 -0300 Subject: [PATCH 060/194] chore: bump app vsns --- apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index a4fbe5673..e5680cfc4 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kafka, [ {description, "EMQX Enterprise Kafka Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, [emqx_bridge_kafka_consumer_sup]}, {applications, [ kernel, From 4af6e3eb6e44fde5e243f5c0320d53c99c38d823 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 13:47:19 -0300 Subject: [PATCH 061/194] refactor: rm unused modules --- .../src/emqx_bridge_pulsar.app.src | 1 - .../src/emqx_bridge_pulsar_app.erl | 14 ----------- .../src/emqx_bridge_pulsar_sup.erl | 24 ------------------- 3 files changed, 39 deletions(-) delete mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl delete mode 100644 apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index cd89f6867..ead7cb715 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -2,7 +2,6 @@ {description, "EMQX Pulsar Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {mod, {emqx_bridge_pulsar_app, []}}, {applications, [ kernel, stdlib, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl deleted file mode 100644 index bedf42cf6..000000000 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_app.erl +++ /dev/null @@ -1,14 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_bridge_pulsar_app). - --behaviour(application). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_bridge_pulsar_sup:start_link(). - -stop(_State) -> - ok. diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl deleted file mode 100644 index ad7e2ae6f..000000000 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_sup.erl +++ /dev/null @@ -1,24 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- --module(emqx_bridge_pulsar_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - --define(SERVER, ?MODULE). - -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -init([]) -> - SupFlags = #{ - strategy => one_for_all, - intensity => 0, - period => 1 - }, - ChildSpecs = [], - {ok, {SupFlags, ChildSpecs}}. From 631863d8432ebc798d4171b819761cb01df54304 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Apr 2023 10:54:58 -0300 Subject: [PATCH 062/194] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_bridge_pulsar.hocon | 2 +- rel/i18n/zh/emqx_bridge_pulsar.hocon | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rel/i18n/emqx_bridge_pulsar.hocon b/rel/i18n/emqx_bridge_pulsar.hocon index 1d2bb4599..205e0271e 100644 --- a/rel/i18n/emqx_bridge_pulsar.hocon +++ b/rel/i18n/emqx_bridge_pulsar.hocon @@ -147,7 +147,7 @@ emqx_bridge_pulsar { "\n" "random: Randomly pick a partition for each message.\n" "roundrobin: Pick each available producer in turn for each message.\n" - "first_key_dispatch: Hash Pulsar message key of the first message in a batch" + "key_dispatch: Hash Pulsar message key of the first message in a batch" " to a partition number." label = "Partition Strategy" } diff --git a/rel/i18n/zh/emqx_bridge_pulsar.hocon b/rel/i18n/zh/emqx_bridge_pulsar.hocon index 3af8652a4..23643060b 100644 --- a/rel/i18n/zh/emqx_bridge_pulsar.hocon +++ b/rel/i18n/zh/emqx_bridge_pulsar.hocon @@ -16,13 +16,13 @@ emqx_bridge_pulsar { } config_enable { - desc = "启用（true）或停用该（false）Pulsar 数据桥接。" + desc = "启用（true）或停用（false）该 Pulsar 数据桥接。" label = "启用或停用" } servers { - desc = "以scheme://host[:port]形式分隔的Pulsar URL列表，" - "供客户端连接使用。支持的方案是 pulsar:// （默认）" + desc = "以逗号分隔的 scheme://host[:port] 格式的 Pulsar URL 列表，" + "支持的 scheme 有 pulsar:// （默认）" "和pulsar+ssl://。默认的端口是6650。" label = "服务员" } @@ -94,7 +94,7 @@ emqx_bridge_pulsar { desc = "设置消息发布时应该如何选择 Pulsar 分区。\n\n" "random: 为每个消息随机选择一个分区。\n" "roundrobin: 依次为每条信息挑选可用的生产商。\n" - "first_key_dispatch: 将一批信息中的第一条信息的Pulsar信息密钥哈希到一个分区编号。" + "key_dispatch: 将一批信息中的第一条信息的Pulsar信息密钥哈希到一个分区编号。" label = "分区选择策略" } From 4f2262129b338a1158774c603351ccdd5ded105c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Apr 2023 11:00:48 -0300 Subject: [PATCH 063/194] refactor: rename `{first_,}key_dispatch` partition strategy option --- apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl | 2 +- .../src/emqx_bridge_pulsar_impl_producer.erl | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index a3e50054e..18faf0e3b 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -83,7 +83,7 @@ fields(producer_opts) -> {pulsar_topic, mk(binary(), #{required => true, desc => ?DESC("producer_pulsar_topic")})}, {strategy, mk( - hoconsc:enum([random, roundrobin, first_key_dispatch]), + hoconsc:enum([random, roundrobin, key_dispatch]), #{default => random, desc => ?DESC("producer_strategy")} )}, {buffer, mk(ref(producer_buffer), #{required => false, desc => ?DESC("producer_buffer")})}, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 72363389e..b86124417 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -28,7 +28,7 @@ }. -type buffer_mode() :: memory | disk | hybrid. -type compression_mode() :: no_compression | snappy | zlib. --type partition_strategy() :: random | roundrobin | first_key_dispatch. +-type partition_strategy() :: random | roundrobin | key_dispatch. -type message_template_raw() :: #{ key := binary(), value := binary() @@ -290,7 +290,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> name => ProducerName, retention_period => RetentionPeriod, ssl_opts => SSLOpts, - strategy => Strategy, + strategy => partition_strategy(Strategy), tcp_opts => [{sndbuf, SendBuffer}] }, ProducerOpts = maps:merge(ReplayQOpts, ProducerOpts0), @@ -394,3 +394,6 @@ get_producer_status(Producers) -> true -> connected; false -> connecting end. + +partition_strategy(key_dispatch) -> first_key_dispatch; +partition_strategy(Strategy) -> Strategy. From cb149ac3458b6048f3559172a1ebeb5801b75000 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Apr 2023 11:02:19 -0300 Subject: [PATCH 064/194] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_bridge_pulsar.hocon | 1 - 1 file changed, 1 deletion(-) diff --git a/rel/i18n/emqx_bridge_pulsar.hocon b/rel/i18n/emqx_bridge_pulsar.hocon index 205e0271e..92294bb75 100644 --- a/rel/i18n/emqx_bridge_pulsar.hocon +++ b/rel/i18n/emqx_bridge_pulsar.hocon @@ -39,7 +39,6 @@ emqx_bridge_pulsar { buffer_mode { desc = "Message buffer mode.\n" - "\n" "memory: Buffer all messages in memory. The messages will be lost" " in case of EMQX node restart\ndisk: Buffer all messages on disk." " The messages on disk are able to survive EMQX node restart.\n" From 4aad5c74545a4bb75df9c771fb32e21327a273d4 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 24 Apr 2023 21:44:21 +0800 Subject: [PATCH 065/194] chore: improve changes --- changes/ce/feat-10457.en.md | 4 ++-- changes/ce/fix-10420.zh.md | 0 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 changes/ce/fix-10420.zh.md diff --git a/changes/ce/feat-10457.en.md b/changes/ce/feat-10457.en.md index d6a44bd53..a11e9b424 100644 --- a/changes/ce/feat-10457.en.md +++ b/changes/ce/feat-10457.en.md @@ -1,4 +1,4 @@ Deprecates the integration with StatsD. -Since StatsD is not used a lot. So we will deprecate it in the next release -and plan to remove it in 5.1 +There seemd to be no user using StatsD integration, so we have decided to hide this feature +for now. We will either remove it based on requirements in the future. diff --git a/changes/ce/fix-10420.zh.md b/changes/ce/fix-10420.zh.md deleted file mode 100644 index e69de29bb..000000000 From 377b1433254c1f047ca689733169f4103ebf1577 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Apr 2023 11:01:33 -0300 Subject: [PATCH 066/194] refactor: split `parse_server` into smaller functions, improve return type to use map --- apps/emqx/src/emqx_schema.erl | 173 ++++++++++++------ apps/emqx/test/emqx_schema_tests.erl | 74 +++++--- .../src/emqx_bridge_cassandra_connector.erl | 11 +- .../emqx_bridge_cassandra_connector_SUITE.erl | 11 +- .../src/emqx_bridge_gcp_pubsub_connector.erl | 2 +- .../src/emqx_bridge_pulsar_impl_producer.erl | 2 +- .../src/emqx_connector_ldap.erl | 12 +- .../src/emqx_connector_mongo.erl | 7 +- .../src/emqx_connector_mysql.erl | 2 +- .../src/emqx_connector_pgsql.erl | 2 +- .../src/emqx_connector_redis.erl | 8 +- .../src/mqtt/emqx_connector_mqtt_schema.erl | 3 +- apps/emqx_statsd/src/emqx_statsd.erl | 2 +- .../test/emqx_ee_bridge_redis_SUITE.erl | 11 +- .../src/emqx_ee_connector_dynamo.erl | 2 +- .../src/emqx_ee_connector_influxdb.erl | 2 +- .../src/emqx_ee_connector_rocketmq.erl | 2 +- .../src/emqx_ee_connector_sqlserver.erl | 2 +- .../src/emqx_ee_connector_tdengine.erl | 2 +- 19 files changed, 220 insertions(+), 110 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 248fdad7f..69f234e47 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -66,6 +66,12 @@ -typerefl_from_string({url/0, emqx_schema, to_url}). -typerefl_from_string({json_binary/0, emqx_schema, to_json_binary}). +-type parsed_server() :: #{ + hostname := string(), + port => port_number(), + scheme => string() +}. + -export([ validate_heap_size/1, user_lookup_fun_tr/2, @@ -2901,10 +2907,7 @@ servers_validator(Opts, Required) -> %% `no_port': by default it's `false', when set to `true', %% a `throw' exception is raised if the port is found. -spec parse_server(undefined | string() | binary(), server_parse_option()) -> - string() - | {string(), port_number()} - | {string(), string()} - | {string(), string(), port_number()}. + undefined | parsed_server(). parse_server(Str, Opts) -> case parse_servers(Str, Opts) of undefined -> @@ -2918,12 +2921,7 @@ parse_server(Str, Opts) -> %% @doc Parse comma separated `host[:port][,host[:port]]' endpoints %% into a list of `{Host, Port}' tuples or just `Host' string. -spec parse_servers(undefined | string() | binary(), server_parse_option()) -> - [ - string() - | {string(), port_number()} - | {string(), string()} - | {string(), string(), port_number()} - ]. + undefined | [parsed_server()]. parse_servers(undefined, _Opts) -> %% should not parse 'undefined' as string, %% not to throw exception either, @@ -2988,55 +2986,112 @@ do_parse_server(Str, Opts) -> ok end, %% do not split with space, there should be no space allowed between host and port - case string:tokens(Str, ":") of - [Scheme, "//" ++ Hostname, Port] -> - NotExpectingPort andalso throw("not_expecting_port_number"), - NotExpectingScheme andalso throw("not_expecting_scheme"), - {check_scheme(Scheme, Opts), check_hostname(Hostname), parse_port(Port)}; - [Scheme, "//" ++ Hostname] -> - NotExpectingScheme andalso throw("not_expecting_scheme"), - case is_integer(DefaultPort) of - true -> - {check_scheme(Scheme, Opts), check_hostname(Hostname), DefaultPort}; - false when NotExpectingPort -> - {check_scheme(Scheme, Opts), check_hostname(Hostname)}; - false -> - throw("missing_port_number") - end; - [Hostname, Port] -> - NotExpectingPort andalso throw("not_expecting_port_number"), - case is_list(DefaultScheme) of - false -> - {check_hostname(Hostname), parse_port(Port)}; - true -> - {DefaultScheme, check_hostname(Hostname), parse_port(Port)} - end; - [Hostname] -> - case is_integer(DefaultPort) orelse NotExpectingPort of - true -> - ok; - false -> - throw("missing_port_number") - end, - case is_list(DefaultScheme) orelse NotExpectingScheme of - true -> - ok; - false -> - throw("missing_scheme") - end, - case {is_integer(DefaultPort), is_list(DefaultScheme)} of - {true, true} -> - {DefaultScheme, check_hostname(Hostname), DefaultPort}; - {true, false} -> - {check_hostname(Hostname), DefaultPort}; - {false, true} -> - {DefaultScheme, check_hostname(Hostname)}; - {false, false} -> - check_hostname(Hostname) - end; - _ -> - throw("bad_host_port") - end. + Tokens = string:tokens(Str, ":"), + Context = #{ + not_expecting_port => NotExpectingPort, + not_expecting_scheme => NotExpectingScheme, + default_port => DefaultPort, + default_scheme => DefaultScheme, + opts => Opts + }, + check_server_parts(Tokens, Context). + +check_server_parts([Scheme, "//" ++ Hostname, Port], Context) -> + #{ + not_expecting_scheme := NotExpectingScheme, + not_expecting_port := NotExpectingPort, + opts := Opts + } = Context, + NotExpectingPort andalso throw("not_expecting_port_number"), + NotExpectingScheme andalso throw("not_expecting_scheme"), + #{ + scheme => check_scheme(Scheme, Opts), + hostname => check_hostname(Hostname), + port => parse_port(Port) + }; +check_server_parts([Scheme, "//" ++ Hostname], Context) -> + #{ + not_expecting_scheme := NotExpectingScheme, + not_expecting_port := NotExpectingPort, + default_port := DefaultPort, + opts := Opts + } = Context, + NotExpectingScheme andalso throw("not_expecting_scheme"), + case is_integer(DefaultPort) of + true -> + #{ + scheme => check_scheme(Scheme, Opts), + hostname => check_hostname(Hostname), + port => DefaultPort + }; + false when NotExpectingPort -> + #{ + scheme => check_scheme(Scheme, Opts), + hostname => check_hostname(Hostname) + }; + false -> + throw("missing_port_number") + end; +check_server_parts([Hostname, Port], Context) -> + #{ + not_expecting_port := NotExpectingPort, + default_scheme := DefaultScheme + } = Context, + NotExpectingPort andalso throw("not_expecting_port_number"), + case is_list(DefaultScheme) of + false -> + #{ + hostname => check_hostname(Hostname), + port => parse_port(Port) + }; + true -> + #{ + scheme => DefaultScheme, + hostname => check_hostname(Hostname), + port => parse_port(Port) + } + end; +check_server_parts([Hostname], Context) -> + #{ + not_expecting_scheme := NotExpectingScheme, + not_expecting_port := NotExpectingPort, + default_port := DefaultPort, + default_scheme := DefaultScheme + } = Context, + case is_integer(DefaultPort) orelse NotExpectingPort of + true -> + ok; + false -> + throw("missing_port_number") + end, + case is_list(DefaultScheme) orelse NotExpectingScheme of + true -> + ok; + false -> + throw("missing_scheme") + end, + case {is_integer(DefaultPort), is_list(DefaultScheme)} of + {true, true} -> + #{ + scheme => DefaultScheme, + hostname => check_hostname(Hostname), + port => DefaultPort + }; + {true, false} -> + #{ + hostname => check_hostname(Hostname), + port => DefaultPort + }; + {false, true} -> + #{ + scheme => DefaultScheme, + hostname => check_hostname(Hostname) + }; + {false, false} -> + #{hostname => check_hostname(Hostname)} + end; +check_server_parts(_Tokens, _Context) -> + throw("bad_host_port"). check_scheme(Str, Opts) -> SupportedSchemes = maps:get(supported_schemes, Opts, []), diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index c13dc8055..cb51aca46 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -219,112 +219,124 @@ parse_server_test_() -> ?T( "single server, binary, no port", ?assertEqual( - [{"localhost", DefaultPort}], + [#{hostname => "localhost", port => DefaultPort}], Parse(<<"localhost">>) ) ), ?T( "single server, string, no port", ?assertEqual( - [{"localhost", DefaultPort}], + [#{hostname => "localhost", port => DefaultPort}], Parse("localhost") ) ), ?T( "single server, list(string), no port", ?assertEqual( - [{"localhost", DefaultPort}], + [#{hostname => "localhost", port => DefaultPort}], Parse(["localhost"]) ) ), ?T( "single server, list(binary), no port", ?assertEqual( - [{"localhost", DefaultPort}], + [#{hostname => "localhost", port => DefaultPort}], Parse([<<"localhost">>]) ) ), ?T( "single server, binary, with port", ?assertEqual( - [{"localhost", 9999}], + [#{hostname => "localhost", port => 9999}], Parse(<<"localhost:9999">>) ) ), ?T( "single server, list(string), with port", ?assertEqual( - [{"localhost", 9999}], + [#{hostname => "localhost", port => 9999}], Parse(["localhost:9999"]) ) ), ?T( "single server, string, with port", ?assertEqual( - [{"localhost", 9999}], + [#{hostname => "localhost", port => 9999}], Parse("localhost:9999") ) ), ?T( "single server, list(binary), with port", ?assertEqual( - [{"localhost", 9999}], + [#{hostname => "localhost", port => 9999}], Parse([<<"localhost:9999">>]) ) ), ?T( "multiple servers, string, no port", ?assertEqual( - [{"host1", DefaultPort}, {"host2", DefaultPort}], + [ + #{hostname => "host1", port => DefaultPort}, + #{hostname => "host2", port => DefaultPort} + ], Parse("host1, host2") ) ), ?T( "multiple servers, binary, no port", ?assertEqual( - [{"host1", DefaultPort}, {"host2", DefaultPort}], + [ + #{hostname => "host1", port => DefaultPort}, + #{hostname => "host2", port => DefaultPort} + ], Parse(<<"host1, host2,,,">>) ) ), ?T( "multiple servers, list(string), no port", ?assertEqual( - [{"host1", DefaultPort}, {"host2", DefaultPort}], + [ + #{hostname => "host1", port => DefaultPort}, + #{hostname => "host2", port => DefaultPort} + ], Parse(["host1", "host2"]) ) ), ?T( "multiple servers, list(binary), no port", ?assertEqual( - [{"host1", DefaultPort}, {"host2", DefaultPort}], + [ + #{hostname => "host1", port => DefaultPort}, + #{hostname => "host2", port => DefaultPort} + ], Parse([<<"host1">>, <<"host2">>]) ) ), ?T( "multiple servers, string, with port", ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], + [#{hostname => "host1", port => 1234}, #{hostname => "host2", port => 2345}], Parse("host1:1234, host2:2345") ) ), ?T( "multiple servers, binary, with port", ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], + [#{hostname => "host1", port => 1234}, #{hostname => "host2", port => 2345}], Parse(<<"host1:1234, host2:2345, ">>) ) ), ?T( "multiple servers, list(string), with port", ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], + [#{hostname => "host1", port => 1234}, #{hostname => "host2", port => 2345}], Parse([" host1:1234 ", "host2:2345"]) ) ), ?T( "multiple servers, list(binary), with port", ?assertEqual( - [{"host1", 1234}, {"host2", 2345}], + [#{hostname => "host1", port => 1234}, #{hostname => "host2", port => 2345}], Parse([<<"host1:1234">>, <<"host2:2345">>]) ) ), @@ -352,7 +364,7 @@ parse_server_test_() -> ?T( "multiple servers without port, mixed list(binary|string)", ?assertEqual( - ["host1", "host2"], + [#{hostname => "host1"}, #{hostname => "host2"}], Parse2([<<"host1">>, "host2"], #{no_port => true}) ) ), @@ -394,14 +406,18 @@ parse_server_test_() -> ?T( "single server map", ?assertEqual( - [{"host1.domain", 1234}], + [#{hostname => "host1.domain", port => 1234}], HoconParse("host1.domain:1234") ) ), ?T( "multiple servers map", ?assertEqual( - [{"host1.domain", 1234}, {"host2.domain", 2345}, {"host3.domain", 3456}], + [ + #{hostname => "host1.domain", port => 1234}, + #{hostname => "host2.domain", port => 2345}, + #{hostname => "host3.domain", port => 3456} + ], HoconParse("host1.domain:1234,host2.domain:2345,host3.domain:3456") ) ), @@ -451,7 +467,7 @@ parse_server_test_() -> ?T( "scheme, hostname and port", ?assertEqual( - {"pulsar+ssl", "host", 6651}, + #{scheme => "pulsar+ssl", hostname => "host", port => 6651}, emqx_schema:parse_server( "pulsar+ssl://host:6651", #{ @@ -464,7 +480,7 @@ parse_server_test_() -> ?T( "scheme and hostname, default port", ?assertEqual( - {"pulsar", "host", 6650}, + #{scheme => "pulsar", hostname => "host", port => 6650}, emqx_schema:parse_server( "pulsar://host", #{ @@ -477,7 +493,7 @@ parse_server_test_() -> ?T( "scheme and hostname, no port", ?assertEqual( - {"pulsar", "host"}, + #{scheme => "pulsar", hostname => "host"}, emqx_schema:parse_server( "pulsar://host", #{ @@ -503,7 +519,7 @@ parse_server_test_() -> ?T( "hostname, default scheme, no default port", ?assertEqual( - {"pulsar", "host"}, + #{scheme => "pulsar", hostname => "host"}, emqx_schema:parse_server( "host", #{ @@ -517,7 +533,7 @@ parse_server_test_() -> ?T( "hostname, default scheme, default port", ?assertEqual( - {"pulsar", "host", 6650}, + #{scheme => "pulsar", hostname => "host", port => 6650}, emqx_schema:parse_server( "host", #{ @@ -544,7 +560,7 @@ parse_server_test_() -> ?T( "hostname, default scheme, defined port", ?assertEqual( - {"pulsar", "host", 6651}, + #{scheme => "pulsar", hostname => "host", port => 6651}, emqx_schema:parse_server( "host:6651", #{ @@ -572,7 +588,7 @@ parse_server_test_() -> ?T( "hostname, default scheme, defined port", ?assertEqual( - {"pulsar", "host", 6651}, + #{scheme => "pulsar", hostname => "host", port => 6651}, emqx_schema:parse_server( "host:6651", #{ @@ -600,9 +616,9 @@ parse_server_test_() -> "multiple hostnames with schemes (1)", ?assertEqual( [ - {"pulsar", "host", 6649}, - {"pulsar+ssl", "other.host", 6651}, - {"pulsar", "yet.another", 6650} + #{scheme => "pulsar", hostname => "host", port => 6649}, + #{scheme => "pulsar+ssl", hostname => "other.host", port => 6651}, + #{scheme => "pulsar", hostname => "yet.another", port => 6650} ], emqx_schema:parse_servers( "pulsar://host:6649, pulsar+ssl://other.host:6651,pulsar://yet.another", diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index cf6ddff9f..d0a1df7a8 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -92,7 +92,7 @@ callback_mode() -> async_if_possible. on_start( InstId, #{ - servers := Servers, + servers := Servers0, keyspace := Keyspace, username := Username, pool_size := PoolSize, @@ -104,9 +104,16 @@ on_start( connector => InstId, config => emqx_utils:redact(Config) }), + Servers = + lists:map( + fun(#{hostname := Host, port := Port}) -> + {Host, Port} + end, + emqx_schema:parse_servers(Servers0, ?DEFAULT_SERVER_OPTION) + ), Options = [ - {nodes, emqx_schema:parse_servers(Servers, ?DEFAULT_SERVER_OPTION)}, + {nodes, Servers}, {username, Username}, {password, emqx_secret:wrap(maps:get(password, Config, ""))}, {keyspace, Keyspace}, diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index f419283a8..452db33a7 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -38,9 +38,14 @@ groups() -> []. cassandra_servers() -> - emqx_schema:parse_servers( - iolist_to_binary([?CASSANDRA_HOST, ":", erlang:integer_to_list(?CASSANDRA_DEFAULT_PORT)]), - #{default_port => ?CASSANDRA_DEFAULT_PORT} + lists:map( + fun(#{hostname := Host, port := Port}) -> + {Host, Port} + end, + emqx_schema:parse_servers( + iolist_to_binary([?CASSANDRA_HOST, ":", erlang:integer_to_list(?CASSANDRA_DEFAULT_PORT)]), + #{default_port => ?CASSANDRA_DEFAULT_PORT} + ) ). init_per_suite(Config) -> diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl index a3f0ef36b..98f3e497d 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl @@ -81,7 +81,7 @@ on_start( %% emulating the emulator behavior %% https://cloud.google.com/pubsub/docs/emulator HostPort = os:getenv("PUBSUB_EMULATOR_HOST", "pubsub.googleapis.com:443"), - {Host, Port} = emqx_schema:parse_server(HostPort, #{default_port => 443}), + #{hostname := Host, port := Port} = emqx_schema:parse_server(HostPort, #{default_port => 443}), PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index b86124417..0b195df66 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -180,7 +180,7 @@ to_bin(B) when is_binary(B) -> format_servers(Servers0) -> Servers1 = emqx_schema:parse_servers(Servers0, ?PULSAR_HOST_OPTIONS), lists:map( - fun({Scheme, Host, Port}) -> + fun(#{scheme := Scheme, hostname := Host, port := Port}) -> Scheme ++ "://" ++ Host ++ ":" ++ integer_to_list(Port) end, Servers1 diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index e2121de22..c3e1db7d3 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -67,7 +67,17 @@ on_start( connector => InstId, config => emqx_utils:redact(Config) }), - Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), + Servers1 = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), + Servers = + lists:map( + fun + (#{hostname := Host, port := Port0}) -> + {Host, Port0}; + (#{hostname := Host}) -> + Host + end, + Servers1 + ), SslOpts = case maps:get(enable, SSL) of true -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index a65a32842..dde8652f0 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -537,4 +537,9 @@ format_hosts(Hosts) -> lists:map(fun format_host/1, Hosts). parse_servers(HoconValue) -> - emqx_schema:parse_servers(HoconValue, ?MONGO_HOST_OPTIONS). + lists:map( + fun(#{hostname := Host, port := Port}) -> + {Host, Port} + end, + emqx_schema:parse_servers(HoconValue, ?MONGO_HOST_OPTIONS) + ). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 45d459e70..b8c1250fe 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -98,7 +98,7 @@ on_start( ssl := SSL } = Config ) -> - {Host, Port} = emqx_schema:parse_server(Server, ?MYSQL_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?MYSQL_HOST_OPTIONS), ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index ddbf9491d..3b2375d04 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -91,7 +91,7 @@ on_start( ssl := SSL } = Config ) -> - {Host, Port} = emqx_schema:parse_server(Server, ?PGSQL_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?PGSQL_HOST_OPTIONS), ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index e2155eb49..32ac77226 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -131,7 +131,13 @@ on_start( _ -> servers end, Servers0 = maps:get(ConfKey, Config), - Servers = [{servers, emqx_schema:parse_servers(Servers0, ?REDIS_HOST_OPTIONS)}], + Servers1 = lists:map( + fun(#{hostname := Host, port := Port}) -> + {Host, Port} + end, + emqx_schema:parse_servers(Servers0, ?REDIS_HOST_OPTIONS) + ), + Servers = [{servers, Servers1}], Database = case Type of cluster -> []; diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index e08804685..2a40980af 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -293,4 +293,5 @@ qos() -> hoconsc:union([emqx_schema:qos(), binary()]). parse_server(Str) -> - emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS). + #{hostname := Host, port := Port} = emqx_schema:parse_server(Str, ?MQTT_HOST_OPTS), + {Host, Port}. diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index c5a7fc1c8..b2d726b07 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -80,7 +80,7 @@ init(Conf) -> flush_time_interval := FlushTimeInterval } = Conf, FlushTimeInterval1 = flush_interval(FlushTimeInterval, SampleTimeInterval), - {Host, Port} = emqx_schema:parse_server(Server, ?SERVER_PARSE_OPTS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?SERVER_PARSE_OPTS), Tags = maps:fold(fun(K, V, Acc) -> [{to_bin(K), to_bin(V)} | Acc] end, [], TagsRaw), Opts = [{tags, Tags}, {host, Host}, {port, Port}, {prefix, <<"emqx">>}], {ok, Pid} = estatsd:start_link(Opts), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl index f0b70d21b..56f932aba 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl @@ -449,9 +449,14 @@ all_test_hosts() -> ). parse_servers(Servers) -> - emqx_schema:parse_servers(Servers, #{ - default_port => 6379 - }). + lists:map( + fun(#{hostname := Host, port := Port}) -> + {Host, Port} + end, + emqx_schema:parse_servers(Servers, #{ + default_port => 6379 + }) + ). redis_connect_ssl_opts(Type) -> maps:merge( diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index ebb86f577..f45f8ca2f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -92,7 +92,7 @@ on_start( }), {Schema, Server} = get_host_schema(to_str(Url)), - {Host, Port} = emqx_schema:parse_server(Server, ?DYNAMO_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?DYNAMO_HOST_OPTIONS), Options = [ {config, #{ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index 700eb2a81..331577486 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -294,7 +294,7 @@ client_config( server := Server } ) -> - {Host, Port} = emqx_schema:parse_server(Server, ?INFLUXDB_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?INFLUXDB_HOST_OPTIONS), [ {host, str(Host)}, {port, Port}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 205359bb8..74fb4eedd 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -105,7 +105,7 @@ on_start( config => redact(Config1) }), Config = maps:merge(default_security_info(), Config1), - {Host, Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS), Server1 = [{Host, Port}], ClientId = client_id(InstanceId), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 70bd76d14..8ea4429d0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -355,7 +355,7 @@ conn_str([], Acc) -> conn_str([{driver, Driver} | Opts], Acc) -> conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]); conn_str([{server, Server} | Opts], Acc) -> - {Host, Port} = emqx_schema:parse_server(Server, ?SQLSERVER_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?SQLSERVER_HOST_OPTIONS), conn_str(Opts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); conn_str([{database, Database} | Opts], Acc) -> conn_str(Opts, ["Database=" ++ str(Database) | Acc]); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index f9ca21ad7..09cbd8db8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -96,7 +96,7 @@ on_start( config => emqx_utils:redact(Config) }), - {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), Options = [ {host, to_bin(Host)}, {port, Port}, From 99f3965f4eb51d503ea63489448ee93c41f588a0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Apr 2023 14:24:42 -0300 Subject: [PATCH 067/194] feat(schema_registry): use rocksdb as table type for protobuf cache --- .../emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src | 2 +- lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src index 87f6e53d0..aa43cf248 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, [emqx_ee_schema_registry_sup]}, {mod, {emqx_ee_schema_registry_app, []}}, {applications, [ diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 59a224fc7..b1453914b 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -179,7 +179,7 @@ create_tables() -> ok = mria:create_table(?PROTOBUF_CACHE_TAB, [ {type, set}, {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, - {storage, disc_only_copies}, + {storage, rocksdb_copies}, {record_name, protobuf_cache}, {attributes, record_info(fields, protobuf_cache)} ]), From 99e892b5c4de941139b480c6de89c0dd62d90857 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Mon, 24 Apr 2023 20:29:38 +0300 Subject: [PATCH 068/194] chore: bump ekka to 0.15.0 ekka 0.15.0 uses mria 0.5.0, which adds several fixes, enhancements and features: - protect `mria:join/1,2` with a global lock - implement new function `mria:sync_transaction/4,3,2`, which waits for a transaction replication to be ready on the local node (if the local node is a replicant) - optimize `mria:running_nodes/0` - optimize `mria:ro_transaction/2` when called on a replicant node. Fixes: EMQX-9588 (#10380), EMQX-9102, EMQX-9152, EMQX-9213 --- apps/emqx/rebar.config | 2 +- changes/ce/fix-10500.en.md | 12 ++++++++++++ mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10500.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 6788b4f40..21b2fd292 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, diff --git a/changes/ce/fix-10500.en.md b/changes/ce/fix-10500.en.md new file mode 100644 index 000000000..730dfb6e5 --- /dev/null +++ b/changes/ce/fix-10500.en.md @@ -0,0 +1,12 @@ +Add several fixes, enhancements and features in Mria: + - protect `mria:join/1,2` with a global lock to prevent conflicts between + two nodes trying to join each other simultaneously + [Mria PR](https://github.com/emqx/mria/pull/137) + - implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until + a transaction is imported to the local node (if the local node is a replicant, otherwise, + it behaves exactly the same as `mria:transaction/3,2`) + [Mria PR](https://github.com/emqx/mria/pull/136) + - optimize `mria:running_nodes/0` + [Mria PR](https://github.com/emqx/mria/pull/135) + - optimize `mria:ro_transaction/2` when called on a replicant node + [Mria PR](https://github.com/emqx/mria/pull/134). diff --git a/mix.exs b/mix.exs index e2230d55d..ca4faabcd 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.14.6", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.0", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, diff --git a/rebar.config b/rebar.config index de520f124..10adb3848 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} From e9fde129131fc3e5f2bbc302cecccded0431b4f9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 24 Apr 2023 15:19:33 -0300 Subject: [PATCH 069/194] test: attempt to fix flaky test Example failure: https://github.com/emqx/emqx/actions/runs/4789177314/jobs/8517116154#step:7:503 --- .../test/emqx_bridge_kafka_impl_consumer_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 08fbf5e15..3d22c0698 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1473,7 +1473,10 @@ do_t_receive_after_recovery(Config) -> ResourceId = resource_id(Config), ?check_trace( begin - {ok, _} = create_bridge(Config), + {ok, _} = create_bridge( + Config, + #{<<"kafka">> => #{<<"offset_reset_policy">> => <<"earliest">>}} + ), ping_until_healthy(Config, _Period = 1_500, _Timeout0 = 24_000), {ok, connected} = emqx_resource_manager:health_check(ResourceId), %% 0) ensure each partition commits its offset so it can From 09b17000c843a87c1e442701a26fb0a3becbed7d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 25 Apr 2023 09:56:40 +0800 Subject: [PATCH 070/194] chore: update changes/ce/feat-10457.en.md Co-authored-by: Zaiming (Stone) Shi --- changes/ce/feat-10457.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10457.en.md b/changes/ce/feat-10457.en.md index a11e9b424..966569a1c 100644 --- a/changes/ce/feat-10457.en.md +++ b/changes/ce/feat-10457.en.md @@ -1,4 +1,4 @@ Deprecates the integration with StatsD. There seemd to be no user using StatsD integration, so we have decided to hide this feature -for now. We will either remove it based on requirements in the future. +for now. We will either remove or revive it based on requirements in the future. From 3f689d0fdf287c38b5f81c9d01f6fe65057f3b7f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 24 Apr 2023 15:27:42 +0800 Subject: [PATCH 071/194] feat: don't do rpc call to check deprecated file --- apps/emqx/priv/bpapi.versions | 1 - apps/emqx_conf/src/emqx_conf_app.erl | 12 +- .../src/proto/emqx_conf_proto_v2.erl | 4 - .../src/proto/emqx_conf_proto_v3.erl | 114 ------------------ 4 files changed, 7 insertions(+), 124 deletions(-) delete mode 100644 apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 11bd4aa77..db4765e3f 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -11,7 +11,6 @@ {emqx_cm,1}. {emqx_conf,1}. {emqx_conf,2}. -{emqx_conf,3}. {emqx_dashboard,1}. {emqx_delayed,1}. {emqx_exhook,1}. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index fd0a56853..fbfb97a79 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -66,7 +66,8 @@ get_override_config_file() -> conf => Conf, tnx_id => TnxId, node => Node, - has_deprecated_file => HasDeprecateFile + has_deprecated_file => HasDeprecateFile, + release => emqx_app:get_release() } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of @@ -180,6 +181,8 @@ copy_override_conf_from_core_node() -> msg => "copy_cluster_conf_from_core_node_success", node => Node, has_deprecated_file => HasDeprecatedFile, + local_release => emqx_app:get_release(), + remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"), data_dir => emqx:data_dir(), tnx_id => TnxId }), @@ -228,13 +231,12 @@ sync_data_from_node(Node) -> error(Error) end. -has_deprecated_file(#{node := Node} = Info) -> +has_deprecated_file(#{conf := Conf} = Info) -> case maps:find(has_deprecated_file, Info) of {ok, HasDeprecatedFile} -> HasDeprecatedFile; error -> %% The old version don't have emqx_config:has_deprecated_file/0 - DataDir = emqx_conf_proto_v2:get_config(Node, [node, data_dir]), - File = filename:join([DataDir, "configs", "cluster-override.conf"]), - emqx_conf_proto_v3:file_exist(Node, File) + %% Conf is not empty if deprecated file is found. + Conf =/= #{} end. diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index 3bcf532f6..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -20,7 +20,6 @@ -export([ introduced_in/0, - deprecated_since/0, sync_data_from_node/1, get_config/2, get_config/3, @@ -42,9 +41,6 @@ introduced_in() -> "5.0.1". -deprecated_since() -> - "5.0.23". - -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl deleted file mode 100644 index 802436f98..000000000 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v3.erl +++ /dev/null @@ -1,114 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_conf_proto_v3). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - sync_data_from_node/1, - get_config/2, - get_config/3, - get_all/1, - - update/3, - update/4, - remove_config/2, - remove_config/3, - - reset/2, - reset/3, - - get_override_config_file/1, - file_exist/2 -]). - --include_lib("emqx/include/bpapi.hrl"). - -introduced_in() -> - "5.0.24". - --spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). -sync_data_from_node(Node) -> - rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. - --spec get_config(node(), emqx_utils_maps:config_key_path()) -> - term() | emqx_rpc:badrpc(). -get_config(Node, KeyPath) -> - rpc:call(Node, emqx, get_config, [KeyPath]). - --spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> - term() | emqx_rpc:badrpc(). -get_config(Node, KeyPath, Default) -> - rpc:call(Node, emqx, get_config, [KeyPath, Default]). - --spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). -get_all(KeyPath) -> - rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). - --spec update( - update_config_key_path(), - emqx_config:update_request(), - emqx_config:update_opts() -) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -update(KeyPath, UpdateReq, Opts) -> - emqx_cluster_rpc:multicall(emqx, update_config, [KeyPath, UpdateReq, Opts]). - --spec update( - node(), - update_config_key_path(), - emqx_config:update_request(), - emqx_config:update_opts() -) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -update(Node, KeyPath, UpdateReq, Opts) -> - rpc:call(Node, emqx, update_config, [KeyPath, UpdateReq, Opts], 5000). - --spec remove_config(update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -remove_config(KeyPath, Opts) -> - emqx_cluster_rpc:multicall(emqx, remove_config, [KeyPath, Opts]). - --spec remove_config(node(), update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -remove_config(Node, KeyPath, Opts) -> - rpc:call(Node, emqx, remove_config, [KeyPath, Opts], 5000). - --spec reset(update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. -reset(KeyPath, Opts) -> - emqx_cluster_rpc:multicall(emqx, reset_config, [KeyPath, Opts]). - --spec reset(node(), update_config_key_path(), emqx_config:update_opts()) -> - {ok, emqx_config:update_result()} - | {error, emqx_config:update_error()} - | emqx_rpc:badrpc(). -reset(Node, KeyPath, Opts) -> - rpc:call(Node, emqx, reset_config, [KeyPath, Opts]). - --spec get_override_config_file([node()]) -> emqx_rpc:multicall_result(). -get_override_config_file(Nodes) -> - rpc:multicall(Nodes, emqx_conf_app, get_override_config_file, [], 20000). - --spec file_exist(node(), string()) -> emqx_rpc:badrpc() | boolean(). -file_exist(Node, File) -> - rpc:call(Node, filelib, is_regular, [File], 5000). From 33c27ac2acf9ff2c6c57b1beef911a360621914b Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 25 Apr 2023 10:50:44 +0800 Subject: [PATCH 072/194] fix(dynamo): use correct default port for different schemas --- .../src/emqx_ee_connector_dynamo.erl | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 01554f90a..3cf7322dc 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -33,10 +33,6 @@ -import(hoconsc, [mk/2, enum/1, ref/2]). --define(DYNAMO_HOST_OPTIONS, #{ - default_port => 80 -}). - %%===================================================================== %% Hocon schema roots() -> @@ -84,8 +80,8 @@ on_start( config => redact(Config) }), - {Schema, Server} = get_host_schema(to_str(Url)), - {Host, Port} = emqx_schema:parse_server(Server, ?DYNAMO_HOST_OPTIONS), + {Schema, Server, DefaultPort} = get_host_info(to_str(Url)), + {Host, Port} = emqx_schema:parse_server(Server, #{default_port => DefaultPort}), Options = [ {config, #{ @@ -226,12 +222,12 @@ to_str(List) when is_list(List) -> to_str(Bin) when is_binary(Bin) -> erlang:binary_to_list(Bin). -get_host_schema("http://" ++ Server) -> - {"http://", Server}; -get_host_schema("https://" ++ Server) -> - {"https://", Server}; -get_host_schema(Server) -> - {"http://", Server}. +get_host_info("http://" ++ Server) -> + {"http://", Server, 80}; +get_host_info("https://" ++ Server) -> + {"https://", Server, 443}; +get_host_info(Server) -> + {"http://", Server, 80}. redact(Data) -> emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). From f84fc6f8b922cc000a45694b0429609e182b77cf Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Apr 2023 11:59:06 +0800 Subject: [PATCH 073/194] fix: can't update authentication when cluster-override.conf --- apps/emqx/src/emqx_config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index c94f25ead..9561263ca 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -333,6 +333,7 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts). init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> + ok = save_schema_mod_and_names(SchemaMod), %% deprecated conf will be removed in 5.1 %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), From d6208d8847906341046fef7f7f49cb4769a271f5 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Apr 2023 14:47:05 +0800 Subject: [PATCH 074/194] test: add test for depreated config file --- apps/emqx/test/emqx_config_SUITE.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 7befd7a16..1704d1476 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -59,3 +59,22 @@ t_fill_default_values(_) -> %% ensure JSON compatible _ = emqx_utils_json:encode(WithDefaults), ok. + +t_init_load(_Config) -> + ConfFile = "./test_emqx.conf", + ok = file:write_file(ConfFile, <<"">>), + ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)), + emqx_config:erase_schema_mod_and_names(), + {ok, DeprecatedFile} = application:get_env(emqx, cluster_override_conf_file), + ?assertEqual(false, filelib:is_regular(DeprecatedFile), DeprecatedFile), + %% Don't has deprecated file + ok = emqx_config:init_load(emqx_schema, [ConfFile]), + ?assertEqual(ExpectRootNames, lists:sort(emqx_config:get_root_names())), + ?assertMatch({ok, #{raw_config := 256}}, emqx:update_config([mqtt, max_topic_levels], 256)), + emqx_config:erase_schema_mod_and_names(), + %% Has deprecated file + ok = file:write_file(DeprecatedFile, <<"{}">>), + ok = emqx_config:init_load(emqx_schema, [ConfFile]), + ?assertEqual(ExpectRootNames, lists:sort(emqx_config:get_root_names())), + ?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)), + ok = file:delete(DeprecatedFile). From 3bb50a5751372be09ace7619684f55441cd2bfb0 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 25 Apr 2023 16:15:28 +0800 Subject: [PATCH 075/194] fix(rocketmq): fix that the update of ACL info not working --- .../src/emqx_ee_connector_rocketmq.erl | 87 +++++++++---------- rel/i18n/emqx_ee_connector_rocketmq.hocon | 22 +++++ 2 files changed, 63 insertions(+), 46 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 29f8ef84d..70a27ef6e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -44,6 +44,17 @@ fields(config) -> binary(), #{default => <<"TopicTest">>, desc => ?DESC(topic)} )}, + {access_key, + mk( + binary(), + #{default => <<>>, desc => ?DESC("access_key")} + )}, + {secret_key, + mk( + binary(), + #{default => <<>>, desc => ?DESC("secret_key")} + )}, + {security_token, mk(binary(), #{default => <<>>, desc => ?DESC(security_token)})}, {sync_timeout, mk( emqx_schema:duration(), @@ -59,39 +70,15 @@ fields(config) -> emqx_schema:bytesize(), #{default => <<"1024KB">>, desc => ?DESC(send_buffer)} )}, - {security_token, mk(binary(), #{default => <<>>, desc => ?DESC(security_token)})} - | relational_fields() + + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. -add_default_username(Fields) -> - lists:map( - fun - ({username, OrigUsernameFn}) -> - {username, add_default_fn(OrigUsernameFn, <<"">>)}; - (Field) -> - Field - end, - Fields - ). - -add_default_fn(OrigFn, Default) -> - fun - (default) -> Default; - (Field) -> OrigFn(Field) - end. - servers() -> Meta = #{desc => ?DESC("servers")}, emqx_schema:servers_sc(Meta, ?ROCKETMQ_HOST_OPTIONS). -relational_fields() -> - Fields = [username, password, auto_reconnect], - Values = lists:filter( - fun({E, _}) -> lists:member(E, Fields) end, - emqx_connector_schema_lib:relational_db_fields() - ), - add_default_username(Values). - %%======================================================================================== %% `emqx_resource' API %%======================================================================================== @@ -102,21 +89,20 @@ is_buffer_supported() -> false. on_start( InstanceId, - #{servers := BinServers, topic := Topic, sync_timeout := SyncTimeout} = Config1 + #{servers := BinServers, topic := Topic, sync_timeout := SyncTimeout} = Config ) -> ?SLOG(info, #{ msg => "starting_rocketmq_connector", connector => InstanceId, - config => redact(Config1) + config => redact(Config) }), - Config = maps:merge(default_security_info(), Config1), Servers = emqx_schema:parse_servers(BinServers, ?ROCKETMQ_HOST_OPTIONS), ClientId = client_id(InstanceId), - ClientCfg = #{acl_info => #{}}, TopicTks = emqx_plugin_libs_rule:preproc_tmpl(Topic), - ProducerOpts = make_producer_opts(Config), + #{acl_info := AclInfo} = ProducerOpts = make_producer_opts(Config), + ClientCfg = #{acl_info => AclInfo}, Templates = parse_template(Config), ProducersMapPID = create_producers_map(ClientId), State = #{ @@ -140,11 +126,21 @@ on_start( Error end. -on_stop(InstanceId, #{client_id := ClientId, producers_map_pid := Pid} = _State) -> +on_stop(InstanceId, #{client_id := ClientId, topic := RawTopic, producers_map_pid := Pid} = _State) -> ?SLOG(info, #{ msg => "stopping_rocketmq_connector", connector => InstanceId }), + + Producers = ets:match(ClientId, {{RawTopic, '$1'}, '$2'}), + lists:foreach( + fun([Topic, Producer]) -> + ets:delete(ClientId, {RawTopic, Topic}), + _ = rocketmq:stop_and_delete_supervised_producers(Producer) + end, + Producers + ), + Pid ! ok, ok = rocketmq:stop_and_delete_supervised_client(ClientId). @@ -276,6 +272,8 @@ client_id(InstanceId) -> redact(Msg) -> emqx_utils:redact(Msg, fun is_sensitive_key/1). +is_sensitive_key(secret_key) -> + true; is_sensitive_key(security_token) -> true; is_sensitive_key(_) -> @@ -283,14 +281,14 @@ is_sensitive_key(_) -> make_producer_opts( #{ - username := Username, - password := Password, + access_key := AccessKey, + secret_key := SecretKey, security_token := SecurityToken, send_buffer := SendBuff, refresh_interval := RefreshInterval } ) -> - ACLInfo = acl_info(Username, Password, SecurityToken), + ACLInfo = acl_info(AccessKey, SecretKey, SecurityToken), #{ tcp_opts => [{sndbuf, SendBuff}], ref_topic_route_interval => RefreshInterval, @@ -299,17 +297,17 @@ make_producer_opts( acl_info(<<>>, <<>>, <<>>) -> #{}; -acl_info(Username, Password, <<>>) when is_binary(Username), is_binary(Password) -> +acl_info(AccessKey, SecretKey, <<>>) when is_binary(AccessKey), is_binary(SecretKey) -> #{ - access_key => Username, - secret_key => Password + access_key => AccessKey, + secret_key => SecretKey }; -acl_info(Username, Password, SecurityToken) when - is_binary(Username), is_binary(Password), is_binary(SecurityToken) +acl_info(AccessKey, SecretKey, SecurityToken) when + is_binary(AccessKey), is_binary(SecretKey), is_binary(SecurityToken) -> #{ - access_key => Username, - secret_key => Password, + access_key => AccessKey, + secret_key => SecretKey, security_token => SecurityToken }; acl_info(_, _, _) -> @@ -342,6 +340,3 @@ get_producers(ClientId, {_, Topic1} = TopicKey, ProducerOpts) -> ets:insert(ClientId, {TopicKey, Producers0}), Producers0 end. - -default_security_info() -> - #{username => <<>>, password => <<>>, security_token => <<>>}. diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index 7f786898e..ddbe3a77b 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -26,6 +26,28 @@ The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" } } + access_key { + desc { + en: """RocketMQ server `accessKey`.""" + zh: """RocketMQ 服务器的 `accessKey`。""" + } + label: { + en: "AccessKey" + zh: "AccessKey" + } + } + + secret_key { + desc { + en: """RocketMQ server `secretKey`.""" + zh: """RocketMQ 服务器的 `secretKey`。""" + } + label: { + en: "SecretKey" + zh: "SecretKey" + } + } + sync_timeout { desc { en: """Timeout of RocketMQ driver synchronous call.""" From 308056f0fc30b28a95a9c352b6167b8fc8f030b1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 25 Apr 2023 18:08:34 +0800 Subject: [PATCH 076/194] feat: improved the storage format of Unicode characters in data files --- apps/emqx/rebar.config | 2 +- changes/ce/feat-10512.en.md | 3 +++ mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-10512.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 21b2fd292..75be887ad 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.3"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/changes/ce/feat-10512.en.md b/changes/ce/feat-10512.en.md new file mode 100644 index 000000000..e6c162742 --- /dev/null +++ b/changes/ce/feat-10512.en.md @@ -0,0 +1,3 @@ +Improved the storage format of Unicode characters in data files, +Now we can store Unicode characters normally. +For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\"" diff --git a/mix.exs b/mix.exs index ca4faabcd..a05804a19 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.3", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 10adb3848..2b6f1b53c 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.3"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 3138e2b3a1cbd772dd1928a111b964fbb1dead23 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 18 Apr 2023 10:19:09 -0300 Subject: [PATCH 077/194] chore: un-hide ocsp stapling config Undoing https://github.com/emqx/emqx/pull/10160 --- apps/emqx/src/emqx_schema.erl | 3 +-- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index b3c5e1778..ba333f111 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2323,8 +2323,6 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> ref("ocsp"), #{ required => false, - %% TODO: remove after e5.0.2 - importance => ?IMPORTANCE_HIDDEN, validator => fun ocsp_inner_validator/1 } )}, @@ -2333,6 +2331,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> boolean(), #{ default => false, + importance => ?IMPORTANCE_MEDIUM, desc => ?DESC("server_ssl_opts_schema_enable_crl_check") } )} diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index dff8ce5a7..15ca29853 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -677,8 +677,12 @@ do_t_update_listener(Config) -> %% no ocsp at first ListenerId = "ssl:default", {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), - ?assertEqual( - undefined, + ?assertMatch( + #{ + <<"enable_ocsp_stapling">> := false, + <<"refresh_http_timeout">> := _, + <<"refresh_interval">> := _ + }, emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), From d6e46dcadb7ae38c110421b10acbb27a0be9dae9 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 25 Apr 2023 14:48:34 +0200 Subject: [PATCH 078/194] ci: Add a script to generate erlang_ls config --- scripts/gen-erlang-ls-config.sh | 116 ++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100755 scripts/gen-erlang-ls-config.sh diff --git a/scripts/gen-erlang-ls-config.sh b/scripts/gen-erlang-ls-config.sh new file mode 100755 index 000000000..0f5bcdecd --- /dev/null +++ b/scripts/gen-erlang-ls-config.sh @@ -0,0 +1,116 @@ +#!/bin/bash +set -eou pipefail +shopt -s nullglob + +# Our fork of rebar3 copies those directories from apps to +# _build/default/lib rather than just making a symlink. Now erlang_ls +# sees the same module twice and if you're just navigating through the +# call stack you accidentally end up editing files in +# _build/default/lib rather than apps + +usage() { + cat < Date: Tue, 25 Apr 2023 10:51:34 -0300 Subject: [PATCH 079/194] chore: tag e5.0.3-alpha.3 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index d1f1a93ae..6d91b2528 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3-alpha.2"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.3"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 4cc4c4ffaad732bcf277fb86352fa19850107289 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 14:26:19 -0300 Subject: [PATCH 080/194] style: format rebar.config file --- apps/emqx_bridge_pulsar/rebar.config | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_pulsar/rebar.config b/apps/emqx_bridge_pulsar/rebar.config index 3b9ae417d..be5f282df 100644 --- a/apps/emqx_bridge_pulsar/rebar.config +++ b/apps/emqx_bridge_pulsar/rebar.config @@ -1,13 +1,14 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {pulsar, {git, "https://github.com/emqx/pulsar-client-erl.git", {tag, "0.8.0"}}} - , {emqx_connector, {path, "../../apps/emqx_connector"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {emqx_bridge, {path, "../../apps/emqx_bridge"}} - ]}. +{deps, [ + {pulsar, {git, "https://github.com/emqx/pulsar-client-erl.git", {tag, "0.8.0"}}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}} +]}. {shell, [ - % {config, "config/sys.config"}, + % {config, "config/sys.config"}, {apps, [emqx_bridge_pulsar]} ]}. From f69ebdcd1a551fe717e6bc2bf9cda4c9abcd59b1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 14:27:07 -0300 Subject: [PATCH 081/194] test(pulsar): teardown tls group --- .../test/emqx_bridge_pulsar_impl_producer_SUITE.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index f86dbc65d..d254b01fc 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -99,7 +99,8 @@ init_per_group(_Group, Config) -> Config. end_per_group(Group, Config) when - Group =:= plain + Group =:= plain; + Group =:= tls -> common_end_per_group(Config), ok; From 56b884ab1776e5e864e14fd9c5835f575ae4ad0d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 14:28:00 -0300 Subject: [PATCH 082/194] style: change docker compose file name --- ...ocker-compose-pulsar-tcp.yaml => docker-compose-pulsar.yaml} | 0 scripts/ct/run.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .ci/docker-compose-file/{docker-compose-pulsar-tcp.yaml => docker-compose-pulsar.yaml} (100%) diff --git a/.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml b/.ci/docker-compose-file/docker-compose-pulsar.yaml similarity index 100% rename from .ci/docker-compose-file/docker-compose-pulsar-tcp.yaml rename to .ci/docker-compose-file/docker-compose-pulsar.yaml diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 307063e84..3a796821c 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -192,7 +192,7 @@ for dep in ${CT_DEPS}; do FILES+=( '.ci/docker-compose-file/docker-compose-opents.yaml' ) ;; pulsar) - FILES+=( '.ci/docker-compose-file/docker-compose-pulsar-tcp.yaml' ) + FILES+=( '.ci/docker-compose-file/docker-compose-pulsar.yaml' ) ;; *) echo "unknown_ct_dependency $dep" From b56a158a545d387f12942e5ebaf54b70059172d7 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 14:29:40 -0300 Subject: [PATCH 083/194] fix(pulsar): fix function return typespec --- .../src/emqx_bridge_pulsar_impl_producer.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 0b195df66..2bd44d16a 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -137,7 +137,10 @@ on_get_status(_InstanceId, State) -> disconnected end. --spec on_query(manager_id(), {send_message, map()}, state()) -> ok | {error, timeout}. +-spec on_query(manager_id(), {send_message, map()}, state()) -> + {ok, term()} + | {error, timeout} + | {error, term()}. on_query(_InstanceId, {send_message, Message}, State) -> #{ producers := Producers, From 19b5ebff813513499b1bb09f5474a62c25825899 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 25 Apr 2023 23:04:42 +0300 Subject: [PATCH 084/194] chore: bump ekka to 0.15.1 ekka 0.15.1 uses mria 0.5.2, which includes the following changes: - fix(mria_membership): call `mria_rlog:role/1` safely - feat: add extra field to ?rlog_sync table (for future use) --- apps/emqx/rebar.config | 2 +- changes/ce/fix-10518.en.md | 6 ++++++ mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-10518.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 21b2fd292..26ff4d1e2 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.0"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, diff --git a/changes/ce/fix-10518.en.md b/changes/ce/fix-10518.en.md new file mode 100644 index 000000000..87d001e91 --- /dev/null +++ b/changes/ce/fix-10518.en.md @@ -0,0 +1,6 @@ +Add the following fixes and features in Mria: + - call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership + gen_server won't crash if RPC to another node fails + [Mria PR](https://github.com/emqx/mria/pull/139) + - Add extra field to ?rlog_sync table to facilitate extending this functionality in future + [Mria PR](https://github.com/emqx/mria/pull/138). diff --git a/mix.exs b/mix.exs index ca4faabcd..5836d5b19 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.0", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.15.1", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.8", override: true}, diff --git a/rebar.config b/rebar.config index 10adb3848..b39d8a868 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.0"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.8"}}} From 687509886e8fab8439637ba5bf0a2a02cd619d81 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 17:37:20 -0300 Subject: [PATCH 085/194] test: rm unused var warning --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 105f1fe75..e2006bc6d 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -425,7 +425,7 @@ t_mongo_date_rule_engine_functions(Config) -> RuleId = <<"rule:t_mongo_date_rule_engine_functions">>, emqx_rule_engine:delete_rule(RuleId), BridgeId = emqx_bridge_resource:bridge_id(Type, Name), - {ok, Rule} = emqx_rule_engine:create_rule( + {ok, _Rule} = emqx_rule_engine:create_rule( #{ id => <<"rule:t_mongo_date_rule_engine_functions">>, sql => SQL, From bc1bdae55d137cbd47a3b67570e47d7e72345320 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Apr 2023 11:27:31 +0800 Subject: [PATCH 086/194] chore: reslove confilt for sync release-50 to master --- .../src/emqx_ee_connector_dynamo.erl | 12 ++++++++---- .../src/emqx_ee_connector_rocketmq.erl | 6 ++++-- .../src/emqx_ee_connector_sqlserver.erl | 5 ++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index b6270b1b6..5eee882ce 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -80,8 +80,10 @@ on_start( config => redact(Config) }), - {Schema, Server} = get_host_schema(to_str(Url)), - #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?DYNAMO_HOST_OPTIONS), + {Schema, Server, DefaultPort} = get_host_info(to_str(Url)), + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, #{ + default_port => DefaultPort + }), Options = [ {config, #{ @@ -142,8 +144,10 @@ on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, ReplyCtx, Stat on_batch_query_async(_InstanceId, Query, _Reply, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. -on_get_status(_InstanceId, #{pool_name := PoolName}) -> - Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), +on_get_status(_InstanceId, #{pool_name := Pool}) -> + Health = emqx_resource_pool:health_check_workers( + Pool, {emqx_ee_connector_dynamo_client, is_connected, []} + ), status_result(Health). status_result(_Status = true) -> connected; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index e831b4f2f..2e1730b52 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -96,8 +96,10 @@ on_start( connector => InstanceId, config => redact(Config) }), - Config = maps:merge(default_security_info(), Config1), - #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?ROCKETMQ_HOST_OPTIONS), + Servers = lists:map( + fun(#{hostname := Host, port := Port}) -> {Host, Port} end, + emqx_schema:parse_servers(BinServers, ?ROCKETMQ_HOST_OPTIONS) + ), ClientId = client_id(InstanceId), TopicTks = emqx_plugin_libs_rule:preproc_tmpl(Topic), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 180d1271c..90d90cb36 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -304,11 +304,10 @@ on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> ), do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). -on_get_status(_InstanceId, #{pool_name := PoolName, resource_opts := ResourceOpts} = _State) -> - RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), +on_get_status(_InstanceId, #{pool_name := PoolName} = _State) -> Health = emqx_resource_pool:health_check_workers( PoolName, - {?MODULE, do_get_status, []}, + {?MODULE, do_get_status, []} ), status_result(Health). From f0cd5c98c731e76e77816d45267ec76f5d33dac1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 26 Apr 2023 13:57:15 +0800 Subject: [PATCH 087/194] chore: split i18n with script --- rel/i18n/emqx_ee_bridge_rocketmq.hocon | 12 +--- rel/i18n/emqx_ee_connector_rocketmq.hocon | 71 ++++++++++---------- rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon | 5 +- rel/i18n/zh/emqx_ee_connector_rocketmq.hocon | 22 +++++- 4 files changed, 63 insertions(+), 47 deletions(-) diff --git a/rel/i18n/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/emqx_ee_bridge_rocketmq.hocon index 4e4f8a99c..e079220b6 100644 --- a/rel/i18n/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_ee_bridge_rocketmq.hocon @@ -34,17 +34,11 @@ local_topic.label: template.desc: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ.
-The template can be any valid string with placeholders, example:
-- ${id}, ${username}, ${clientid}, ${timestamp}
-- {\"id\" : ${id}, \"username\" : ${username}}""" + The template can be any valid string with placeholders, example:
+ - ${id}, ${username}, ${clientid}, ${timestamp}
+ - {"id" : ${id}, "username" : ${username}}""" template.label: """Template""" -config_enable.desc: -"""Enable or disable this bridge""" - -config_enable.label: -"""Enable Or Disable Bridge""" - } diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index 661cbe249..d3d59a389 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -1,52 +1,53 @@ emqx_ee_connector_rocketmq { +access_key.desc: +"""RocketMQ server `accessKey`.""" + +access_key.label: +"""AccessKey""" + +refresh_interval.desc: +"""RocketMQ Topic Route Refresh Interval.""" + +refresh_interval.label: +"""Topic Route Refresh Interval""" + +secret_key.desc: +"""RocketMQ server `secretKey`.""" + +secret_key.label: +"""SecretKey""" + +security_token.desc: +"""RocketMQ Server Security Token""" + +security_token.label: +"""Security Token""" + +send_buffer.desc: +"""The socket send buffer size of the RocketMQ driver client.""" + +send_buffer.label: +"""Send Buffer Size""" + servers.desc: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" -servers.label: +servers.label: """Server Host""" -topic.desc: -"""RocketMQ Topic""" - -topic.label: -"""RocketMQ Topic""" - -access_key.desc: -"""RocketMQ server `accessKey`.""" - -access_key.label: -"""AccessKey""" - -secret_key.desc: -"""RocketMQ server `secretKey`.""" -secret_key.label: -"""SecretKey""" - sync_timeout.desc: """Timeout of RocketMQ driver synchronous call.""" -sync_timeout.label: +sync_timeout.label: """Sync Timeout""" - -refresh_interval.desc: -"""RocketMQ Topic Route Refresh Interval.""" - -refresh_interval.label: -"""Topic Route Refresh Interval""" -send_buffer.desc: -"""The socket send buffer size of the RocketMQ driver client.""" +topic.desc: +"""RocketMQ Topic""" -send_buffer.label: -"""Send Buffer Size""" - -security_token.desc: -"""RocketMQ Server Security Token""" - -security_token.label: -"""Security Token""" +topic.label: +"""RocketMQ Topic""" } diff --git a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon index 924004361..445a54232 100644 --- a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon @@ -32,7 +32,10 @@ local_topic.label: """本地 Topic""" template.desc: -"""模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" +"""模板, 默认为空，为空时将会将整个消息转发给 RocketMQ。
+ 模板可以是任意带有占位符的合法字符串, 例如:
+ - ${id}, ${username}, ${clientid}, ${timestamp}
+ - {"id" : ${id}, "username" : ${username}}""" template.label: """模板""" diff --git a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon index d32e6ea01..58a1f7ddb 100644 --- a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon @@ -1,11 +1,23 @@ emqx_ee_connector_rocketmq { +access_key.desc: +"""RocketMQ 服务器的 `accessKey`。""" + +access_key.label: +"""AccessKey""" + refresh_interval.desc: """RocketMQ 主题路由更新间隔。""" refresh_interval.label: """主题路由更新间隔""" +secret_key.desc: +"""RocketMQ 服务器的 `secretKey`。""" + +secret_key.label: +"""SecretKey""" + security_token.desc: """RocketMQ 服务器安全令牌""" @@ -18,14 +30,20 @@ send_buffer.desc: send_buffer.label: """发送消息的缓冲区大小""" -server.desc: +servers.desc: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" -server.label: +servers.label: """服务器地址""" +sync_timeout.desc: +"""RocketMQ 驱动同步调用的超时时间。""" + +sync_timeout.label: +"""同步调用超时时间""" + topic.desc: """RocketMQ 主题""" From e467e082f098ad567c9217c7725a7b5017387b04 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 26 Apr 2023 14:44:08 +0800 Subject: [PATCH 088/194] fix(dynamo): remove all async callbacks of the Dynamo connector --- .../src/emqx_ee_connector_dynamo.erl | 28 +++---------------- .../src/emqx_ee_connector_dynamo_client.erl | 13 ++++----- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 3cf7322dc..4b8392725 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -22,8 +22,6 @@ on_stop/2, on_query/3, on_batch_query/3, - on_query_async/4, - on_batch_query_async/4, on_get_status/2 ]). @@ -60,7 +58,7 @@ fields(config) -> %% `emqx_resource' API %%======================================================================================== -callback_mode() -> async_if_possible. +callback_mode() -> always_sync. is_buffer_supported() -> false. @@ -115,32 +113,15 @@ on_stop(InstanceId, #{poolname := PoolName} = _State) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstanceId, Query, State) -> - do_query(InstanceId, Query, sync, State). - -on_query_async(InstanceId, Query, ReplyCtx, State) -> - do_query( - InstanceId, - Query, - {async, ReplyCtx}, - State - ). + do_query(InstanceId, Query, State). %% we only support batch insert on_batch_query(InstanceId, [{send_message, _} | _] = Query, State) -> - do_query(InstanceId, Query, sync, State); + do_query(InstanceId, Query, State); on_batch_query(_InstanceId, Query, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. %% we only support batch insert -on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, ReplyCtx, State) -> - do_query( - InstanceId, - Query, - {async, ReplyCtx}, - State - ); -on_batch_query_async(_InstanceId, Query, _Reply, _State) -> - {error, {unrecoverable_error, {invalid_request, Query}}}. on_get_status(_InstanceId, #{poolname := Pool}) -> Health = emqx_plugin_libs_pool:health_check_ecpool_workers( @@ -158,7 +139,6 @@ status_result(_Status = false) -> connecting. do_query( InstanceId, Query, - ApplyMode, #{poolname := PoolName, templates := Templates, table := Table} = State ) -> ?TRACE( @@ -168,7 +148,7 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {emqx_ee_connector_dynamo_client, query, [ApplyMode, Table, Query, Templates]}, + {emqx_ee_connector_dynamo_client, query, [Table, Query, Templates]}, no_handover ), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl index 0340655b4..8f27497fa 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl @@ -9,7 +9,6 @@ -export([ start_link/1, is_connected/1, - query/5, query/4 ]). @@ -28,22 +27,22 @@ -export([execute/2]). -endif. +%% The default timeout for DynamoDB REST API calls is 10 seconds, +%% but this value for `gen_server:call` is 5s, +%% so we should pass the timeout to `gen_server:call` +-define(HEALTH_CHECK_TIMEOUT, 10000). + %%%=================================================================== %%% API %%%=================================================================== is_connected(Pid) -> try - gen_server:call(Pid, is_connected) + gen_server:call(Pid, is_connected, ?HEALTH_CHECK_TIMEOUT) catch _:_ -> false end. -query(Pid, sync, Table, Query, Templates) -> - query(Pid, Table, Query, Templates); -query(Pid, {async, ReplyCtx}, Table, Query, Templates) -> - gen_server:cast(Pid, {query, Table, Query, Templates, ReplyCtx}). - query(Pid, Table, Query, Templates) -> gen_server:call(Pid, {query, Table, Query, Templates}, infinity). From 6706fd90e1f1a2214a863f076a8fc37bd1fe83dd Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 26 Apr 2023 16:10:35 +0800 Subject: [PATCH 089/194] fix(rocketmq): keep sensitive data safe in rocketmq logs and state --- .../src/emqx_ee_connector_rocketmq.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 70a27ef6e..73f89491b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -112,18 +112,19 @@ on_start( sync_timeout => SyncTimeout, templates => Templates, producers_map_pid => ProducersMapPID, - producers_opts => ProducerOpts + producers_opts => emqx_secret:wrap(ProducerOpts) }, case rocketmq:ensure_supervised_client(ClientId, Servers, ClientCfg) of {ok, _Pid} -> {ok, State}; - {error, _Reason} = Error -> + {error, Reason0} -> + Reason = redact(Reason0), ?tp( rocketmq_connector_start_failed, - #{error => _Reason} + #{error => Reason} ), - Error + {error, Reason} end. on_stop(InstanceId, #{client_id := ClientId, topic := RawTopic, producers_map_pid := Pid} = _State) -> @@ -220,7 +221,7 @@ safe_do_produce(InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, R produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout) catch _Type:Reason -> - {error, {unrecoverable_error, Reason}} + {error, {unrecoverable_error, redact(Reason)}} end. produce(_InstanceId, QueryFunc, Producers, Data, RequestTimeout) -> @@ -335,7 +336,7 @@ get_producers(ClientId, {_, Topic1} = TopicKey, ProducerOpts) -> _ -> ProducerGroup = iolist_to_binary([atom_to_list(ClientId), "_", Topic1]), {ok, Producers0} = rocketmq:ensure_supervised_producers( - ClientId, ProducerGroup, Topic1, ProducerOpts + ClientId, ProducerGroup, Topic1, emqx_secret:unwrap(ProducerOpts) ), ets:insert(ClientId, {TopicKey, Producers0}), Producers0 From 5ed3c3a92cd78ecfd8feeafe953a84f840f219fb Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 17 Apr 2023 14:18:19 +0200 Subject: [PATCH 090/194] perf(config): eliminate make_ref() calls in config get calls --- apps/emqx/src/emqx_config.erl | 20 +++++++++----------- apps/emqx_utils/src/emqx_utils.app.src | 2 +- apps/emqx_utils/src/emqx_utils_maps.erl | 9 ++++----- changes/ce/perf-10417.en.md | 4 ++++ 4 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 changes/ce/perf-10417.en.md diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 9561263ca..8ec8c11ab 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -103,6 +103,8 @@ -define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]). -define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]). +-define(CONFIG_NOT_FOUND_MAGIC, '$0tFound'). + -export_type([ update_request/0, raw_config/0, @@ -164,9 +166,8 @@ get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). -spec find(emqx_utils_maps:config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> - Ref = make_ref(), - case do_get(?CONF, [], Ref) of - Ref -> {not_found, []}; + case do_get(?CONF, [], ?CONFIG_NOT_FOUND_MAGIC) of + ?CONFIG_NOT_FOUND_MAGIC -> {not_found, []}; Res -> {ok, Res} end; find(KeyPath) -> @@ -179,9 +180,8 @@ find(KeyPath) -> -spec find_raw(emqx_utils_maps:config_key_path()) -> {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_raw([]) -> - Ref = make_ref(), - case do_get_raw([], Ref) of - Ref -> {not_found, []}; + case do_get_raw([], ?CONFIG_NOT_FOUND_MAGIC) of + ?CONFIG_NOT_FOUND_MAGIC -> {not_found, []}; Res -> {ok, Res} end; find_raw(KeyPath) -> @@ -666,11 +666,9 @@ do_get_raw(Path, Default) -> do_get(?RAW_CONF, Path, Default). do_get(Type, KeyPath) -> - Ref = make_ref(), - Res = do_get(Type, KeyPath, Ref), - case Res =:= Ref of - true -> error({config_not_found, KeyPath}); - false -> Res + case do_get(Type, KeyPath, ?CONFIG_NOT_FOUND_MAGIC) of + ?CONFIG_NOT_FOUND_MAGIC -> error({config_not_found, KeyPath}); + Res -> Res end. do_get(Type, [], Default) -> diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index eb6371411..dff55bc86 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.0"}, + {vsn, "5.0.1"}, {modules, [ emqx_utils, emqx_utils_api, diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index 6bec32ae3..d1c3ed649 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -41,14 +41,13 @@ -type config_key_path() :: [config_key()]. -type convert_fun() :: fun((...) -> {K1 :: any(), V1 :: any()} | drop). +-define(CONFIG_NOT_FOUND_MAGIC, '$0tFound'). %%----------------------------------------------------------------- -spec deep_get(config_key_path(), map()) -> term(). deep_get(ConfKeyPath, Map) -> - Ref = make_ref(), - Res = deep_get(ConfKeyPath, Map, Ref), - case Res =:= Ref of - true -> error({config_not_found, ConfKeyPath}); - false -> Res + case deep_get(ConfKeyPath, Map, ?CONFIG_NOT_FOUND_MAGIC) of + ?CONFIG_NOT_FOUND_MAGIC -> error({config_not_found, ConfKeyPath}); + Res -> Res end. -spec deep_get(config_key_path(), map(), term()) -> term(). diff --git a/changes/ce/perf-10417.en.md b/changes/ce/perf-10417.en.md new file mode 100644 index 000000000..b9f1217c9 --- /dev/null +++ b/changes/ce/perf-10417.en.md @@ -0,0 +1,4 @@ +Improve get config performance + +eliminate make_ref calls + From 9eccfa5cdf2f2c72e17d1bedc4fef0c2254054ac Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 26 Apr 2023 17:03:01 +0800 Subject: [PATCH 091/194] fix(dynamo): fix test case errors --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 3b07acbe0..88bce879e 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -291,7 +291,7 @@ t_setup_via_config_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(dynamo_connector_query_return, Trace0), - ?assertMatch([#{result := ok}], Trace), + ?assertMatch([#{result := {ok, _}}], Trace), ok end ), @@ -328,7 +328,7 @@ t_setup_via_http_api_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(dynamo_connector_query_return, Trace0), - ?assertMatch([#{result := ok}], Trace), + ?assertMatch([#{result := {ok, _}}], Trace), ok end ), From 54c1a2b06d23ac38cdfab64f0ab03372dbc2d951 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 25 Apr 2023 17:21:36 +0200 Subject: [PATCH 092/194] ci: add performance test workflow --- .github/workflows/performance_test.yaml | 125 ++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 .github/workflows/performance_test.yaml diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml new file mode 100644 index 000000000..e8a2a321e --- /dev/null +++ b/.github/workflows/performance_test.yaml @@ -0,0 +1,125 @@ +name: Performance Test Suite + +on: + push: + branches: + - 'perf/**' + schedule: + - cron: '0 1 * * *' + workflow_dispatch: + inputs: + ref: + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu20.04 + outputs: + BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }} + PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.ref }} + - id: prepare + run: | + echo "EMQX_NAME=emqx" >> $GITHUB_ENV + echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Build deb package + run: | + make ${EMQX_NAME}-pkg + ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg + - name: Get package file name + id: package_file + run: | + echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT + - uses: actions/upload-artifact@v3 + with: + name: emqx-ubuntu20.04 + path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }} + + tf_emqx_perf_test: + runs-on: ubuntu-latest + needs: + - prepare + env: + TF_VAR_bench_id: ${{ needs.prepare.outputs.BENCH_ID }} + TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} + TF_VAR_test_duration_seconds: 300 + TF_VAR_grafana_api_key: ${{ secrets.TF_EMQX_PERF_TEST_GRAFANA_API_KEY }} + TF_AWS_REGION: eu-north-1 + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} + aws-region: eu-north-1 + - name: Checkout tf-emqx-performance-test + uses: actions/checkout@v3 + with: + repository: emqx/tf-emqx-performance-test + path: tf-emqx-performance-test + - uses: actions/download-artifact@v3 + with: + name: emqx-ubuntu20.04 + path: tf-emqx-performance-test/ + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_wrapper: false + - name: terraform init + working-directory: ./tf-emqx-performance-test + run: | + terraform init + - name: terraform apply + working-directory: ./tf-emqx-performance-test + run: | + terraform apply -auto-approve + - name: Wait for test results + working-directory: ./tf-emqx-performance-test + id: test-results + run: | + sleep $TF_VAR_test_duration_seconds + until aws s3api head-object --bucket tf-emqx-performance-test --key "$TF_VAR_bench_id/DONE" > /dev/null 2>&1 + do + echo 'waiting' + sleep 10 + done + aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/metrics.json" ./ + aws s3 cp "s3://tf-emqx-performance-test/$TF_VAR_bench_id/stats.json" ./ + echo MESSAGES_DELIVERED=$(cat metrics.json | jq '[.[]."messages.delivered"] | add') >> $GITHUB_OUTPUT + echo MESSAGES_DROPPED=$(cat metrics.json | jq '[.[]."messages.dropped"] | add') >> $GITHUB_OUTPUT + - name: Send notification to Slack + if: success() + uses: slackapi/slack-github-action@v1.23.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + with: + payload: | + {"text": "EMQX performance test completed.\nMessages delivered: ${{ steps.test-results.outputs.MESSAGES_DELIVERED }}.\nMessages dropped: ${{ steps.test-results.outputs.MESSAGES_DROPPED }}.\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - name: terraform destroy + if: always() + working-directory: ./tf-emqx-performance-test + run: | + terraform destroy -auto-approve + - uses: actions/upload-artifact@v3 + if: success() + with: + name: test-results + path: "./tf-emqx-performance-test/*.json" + - uses: actions/upload-artifact@v3 + if: always() + with: + name: terraform + path: | + ./tf-emqx-performance-test/.terraform + ./tf-emqx-performance-test/*.tfstate From abf150518c1e9188bc2ac287a69baabcbd15ac5f Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 26 Apr 2023 12:16:16 +0200 Subject: [PATCH 093/194] fix(test): avoid port collision Use OS selected free port to avoid port collision among the test runs. --- apps/emqx/test/emqx_common_test_helpers.erl | 34 +++++++++++++++- apps/emqx/test/emqx_listeners_SUITE.erl | 39 ++++++++++++------- .../test/emqx_quic_multistreams_SUITE.erl | 13 +------ 3 files changed, 60 insertions(+), 26 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 8603be879..ac03f4660 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -59,7 +59,8 @@ read_schema_configs/2, render_config_file/2, wait_for/4, - wait_mqtt_payload/1 + wait_mqtt_payload/1, + select_free_port/1 ]). -export([ @@ -1242,3 +1243,34 @@ get_or_spawn_janitor() -> on_exit(Fun) -> Janitor = get_or_spawn_janitor(), ok = emqx_test_janitor:push_on_exit_callback(Janitor, Fun). + +%%------------------------------------------------------------------------------- +%% Select a free transport port from the OS +%%------------------------------------------------------------------------------- +%% @doc get unused port from OS +-spec select_free_port(tcp | udp | ssl | quic) -> inets:port_number(). +select_free_port(tcp) -> + select_free_port(gen_tcp, listen); +select_free_port(udp) -> + select_free_port(gen_udp, open); +select_free_port(ssl) -> + select_free_port(tcp); +select_free_port(quic) -> + select_free_port(udp). + +select_free_port(GenModule, Fun) when + GenModule == gen_tcp orelse + GenModule == gen_udp +-> + {ok, S} = GenModule:Fun(0, [{reuseaddr, true}]), + {ok, Port} = inet:port(S), + ok = GenModule:close(S), + case os:type() of + {unix, darwin} -> + %% in MacOS, still get address_in_use after close port + timer:sleep(500); + _ -> + skip + end, + ct:pal("Select free OS port: ~p", [Port]), + Port. diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 107f3d4e7..f0c18fa30 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -47,13 +47,14 @@ init_per_testcase(Case, Config) when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp -> catch emqx_config_handler:stop(), + Port = emqx_common_test_helpers:select_free_port(tcp), {ok, _} = emqx_config_handler:start_link(), PrevListeners = emqx_config:get([listeners], #{}), PureListeners = remove_default_limiter(PrevListeners), PureListeners2 = PureListeners#{ tcp => #{ listener_test => #{ - bind => {"127.0.0.1", 9999}, + bind => {"127.0.0.1", Port}, max_connections => 4321, limiter => #{} } @@ -63,19 +64,20 @@ init_per_testcase(Case, Config) when ok = emqx_listeners:start(), [ - {prev_listener_conf, PrevListeners} + {prev_listener_conf, PrevListeners}, + {tcp_port, Port} | Config ]; init_per_testcase(t_wss_conn, Config) -> catch emqx_config_handler:stop(), + Port = emqx_common_test_helpers:select_free_port(ssl), {ok, _} = emqx_config_handler:start_link(), - PrevListeners = emqx_config:get([listeners], #{}), PureListeners = remove_default_limiter(PrevListeners), PureListeners2 = PureListeners#{ wss => #{ listener_test => #{ - bind => {{127, 0, 0, 1}, 9998}, + bind => {{127, 0, 0, 1}, Port}, limiter => #{}, ssl_options => #{ cacertfile => ?CERTS_PATH("cacert.pem"), @@ -89,7 +91,8 @@ init_per_testcase(t_wss_conn, Config) -> ok = emqx_listeners:start(), [ - {prev_listener_conf, PrevListeners} + {prev_listener_conf, PrevListeners}, + {wss_port, Port} | Config ]; init_per_testcase(_, Config) -> @@ -171,20 +174,30 @@ t_restart_listeners_with_hibernate_after_disabled(_Config) -> ok = emqx_listeners:stop(), emqx_config:put([listeners], OldLConf). -t_max_conns_tcp(_) -> +t_max_conns_tcp(Config) -> %% Note: Using a string representation for the bind address like %% "127.0.0.1" does not work - ?assertEqual(4321, emqx_listeners:max_conns('tcp:listener_test', {{127, 0, 0, 1}, 9999})). + ?assertEqual( + 4321, + emqx_listeners:max_conns('tcp:listener_test', {{127, 0, 0, 1}, ?config(tcp_port, Config)}) + ). -t_current_conns_tcp(_) -> - ?assertEqual(0, emqx_listeners:current_conns('tcp:listener_test', {{127, 0, 0, 1}, 9999})). +t_current_conns_tcp(Config) -> + ?assertEqual( + 0, + emqx_listeners:current_conns('tcp:listener_test', { + {127, 0, 0, 1}, ?config(tcp_port, Config) + }) + ). -t_wss_conn(_) -> - {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), +t_wss_conn(Config) -> + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, ?config(wss_port, Config), [{verify, verify_none}], 1000 + ), ok = ssl:close(Socket). t_quic_conn(Config) -> - Port = 24568, + Port = emqx_common_test_helpers:select_free_port(quic), DataDir = ?config(data_dir, Config), SSLOpts = #{ password => ?SERVER_KEY_PASSWORD, @@ -207,7 +220,7 @@ t_quic_conn(Config) -> emqx_listeners:stop_listener(quic, ?FUNCTION_NAME, #{bind => Port}). t_ssl_password_cert(Config) -> - Port = 24568, + Port = emqx_common_test_helpers:select_free_port(ssl), DataDir = ?config(data_dir, Config), SSLOptsPWD = #{ password => ?SERVER_KEY_PASSWORD, diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 4afd965bd..b55a28206 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -2026,18 +2026,7 @@ stop_emqx() -> %% select a random port picked by OS -spec select_port() -> inet:port_number(). select_port() -> - {ok, S} = gen_udp:open(0, [{reuseaddr, true}]), - {ok, {_, Port}} = inet:sockname(S), - gen_udp:close(S), - case os:type() of - {unix, darwin} -> - %% in MacOS, still get address_in_use after close port - timer:sleep(500); - _ -> - skip - end, - ct:pal("select port: ~p", [Port]), - Port. + emqx_common_test_helpers:select_free_port(quic). -spec via_stream({quic, quicer:connection_handle(), quicer:stream_handle()}) -> quicer:stream_handle(). From 1c4f4037a5374e7d8416b51447f2a23032d11066 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Apr 2023 13:41:42 +0200 Subject: [PATCH 094/194] test(ct/run.sh): remove the trailing / in app name --- scripts/ct/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index a85aa36af..f8ed3dff0 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -47,7 +47,7 @@ while [ "$#" -gt 0 ]; do exit 0 ;; --app) - WHICH_APP="$2" + WHICH_APP="${2%/}" shift 2 ;; --only-up) From a79c741ee5e12d578e1a1f04a5a1b79614fba4ac Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 25 Apr 2023 18:02:10 +0200 Subject: [PATCH 095/194] build: fix docdir --- rebar.config.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 3c863046f..dc445db54 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -509,7 +509,7 @@ etc_overlay(ReleaseType, Edition) -> [ {mkdir, "etc/"}, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, - {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.example", "etc/emqx.conf.example"} + {copy, "_build/docgen/" ++ profile() ++ "/emqx.conf.example", "etc/emqx.conf.example"} ] ++ lists:map( fun @@ -646,5 +646,15 @@ list_dir(Dir) -> [] end. -name(ce) -> "emqx"; -name(ee) -> "emqx-enterprise". +profile() -> + case os:getenv("PROFILE") of + Profile = "emqx-enterprise" ++ _ -> + Profile; + Profile = "emqx" ++ _ -> + Profile; + false -> + "emqx-enterprise"; + Profile -> + io:format(standard_error, "ERROR: bad_PROFILE ~p~n", [Profile]), + exit(bad_PROFILE) + end. From d78312e10ebb7335ac0fd675b3f30aeef355ec64 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 25 Apr 2023 17:58:23 -0300 Subject: [PATCH 096/194] test(resource): fix flaky test --- .../src/emqx_resource_buffer_worker.erl | 2 +- .../test/emqx_resource_SUITE.erl | 56 ++++++++++++++++--- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 2e2cd5631..7cb7f8198 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1466,7 +1466,7 @@ mark_inflight_items_as_retriable(Data, WorkerMRef) -> end ), _NumAffected = ets:select_replace(InflightTID, MatchSpec), - ?tp(buffer_worker_worker_down_update, #{num_affected => _NumAffected}), + ?tp(buffer_worker_async_agent_down, #{num_affected => _NumAffected}), ok. %% used to update a batch after dropping expired individual queries. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index f8ddd56b5..34781df6c 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1766,12 +1766,6 @@ t_async_pool_worker_death(_Config) -> ?assertEqual(NumReqs, Inflight0), %% grab one of the worker pids and kill it - {ok, SRef1} = - snabbkaffe:subscribe( - ?match_event(#{?snk_kind := buffer_worker_worker_down_update}), - NumBufferWorkers, - 10_000 - ), {ok, #{pid := Pid0}} = emqx_resource:simple_sync_query(?ID, get_state), MRef = monitor(process, Pid0), ct:pal("will kill ~p", [Pid0]), @@ -1785,13 +1779,27 @@ t_async_pool_worker_death(_Config) -> end, %% inflight requests should have been marked as retriable - {ok, _} = snabbkaffe:receive_events(SRef1), + wait_until_all_marked_as_retriable(NumReqs), Inflight1 = emqx_resource_metrics:inflight_get(?ID), ?assertEqual(NumReqs, Inflight1), - ok + NumReqs end, - [] + fun(NumReqs, Trace) -> + Events = ?of_kind(buffer_worker_async_agent_down, Trace), + %% At least one buffer worker should have marked its + %% requests as retriable. If a single one has + %% received all requests, that's all we got. + ?assertMatch([_ | _], Events), + %% All requests distributed over all buffer workers + %% should have been marked as retriable, by the time + %% the inflight has been drained. + ?assertEqual( + NumReqs, + lists:sum([N || #{num_affected := N} <- Events]) + ), + ok + end ), ok. @@ -3017,3 +3025,33 @@ trace_between_span(Trace0, Marker) -> {Trace1, [_ | _]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := {complete, _}}, Trace0), {[_ | _], [_ | Trace2]} = ?split_trace_at(#{?snk_kind := Marker, ?snk_span := start}, Trace1), Trace2. + +wait_until_all_marked_as_retriable(NumExpected) when NumExpected =< 0 -> + ok; +wait_until_all_marked_as_retriable(NumExpected) -> + Seen = #{}, + do_wait_until_all_marked_as_retriable(NumExpected, Seen). + +do_wait_until_all_marked_as_retriable(NumExpected, _Seen) when NumExpected =< 0 -> + ok; +do_wait_until_all_marked_as_retriable(NumExpected, Seen) -> + Res = ?block_until( + #{?snk_kind := buffer_worker_async_agent_down, ?snk_meta := #{pid := P}} when + not is_map_key(P, Seen), + 10_000 + ), + case Res of + {timeout, Evts} -> + ct:pal("events so far:\n ~p", [Evts]), + ct:fail("timeout waiting for events"); + {ok, #{num_affected := NumAffected, ?snk_meta := #{pid := Pid}}} -> + ct:pal("affected: ~p; pid: ~p", [NumAffected, Pid]), + case NumAffected >= NumExpected of + true -> + ok; + false -> + do_wait_until_all_marked_as_retriable(NumExpected - NumAffected, Seen#{ + Pid => true + }) + end + end. From c83d630c97763cecc6945f4eae99d61b88cecf6a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Apr 2023 14:30:08 +0200 Subject: [PATCH 097/194] fix(cassandra): ensure async calls return connection pid so the buffer worker can monitor it and perform retries if the connection restarted --- .../test/emqx_ee_bridge_cassa_SUITE.erl | 6 ++--- .../src/emqx_ee_connector_cassa.erl | 23 +++++++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 3e442a926..2e3510aed 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -404,7 +404,7 @@ t_setup_via_config_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - ?assertMatch([#{result := ok}], Trace), + ?assertMatch([#{result := {ok, _Pid}}], Trace), ok end ), @@ -443,7 +443,7 @@ t_setup_via_http_api_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - ?assertMatch([#{result := ok}], Trace), + ?assertMatch([#{result := {ok, _Pid}}], Trace), ok end ), @@ -603,7 +603,7 @@ t_missing_data(Config) -> fun(Trace0) -> %% 1. ecql driver will return `ok` first in async query Trace = ?of_kind(cassandra_connector_query_return, Trace0), - ?assertMatch([#{result := ok}], Trace), + ?assertMatch([#{result := {ok, _Pid}}], Trace), %% 2. then it will return an error in callback function Trace1 = ?of_kind(handle_async_reply, Trace0), ?assertMatch([#{result := {error, {8704, _}}}], Trace1), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index 86b908038..c4f3e9b87 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -278,7 +278,7 @@ proc_cql_params(query, SQL, Params, _State) -> exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when Type == query; Type == prepared_query -> - case ecpool:pick_and_do(PoolName, {?MODULE, Type, [Async, PreparedKey, Data]}, no_handover) of + case exec(PoolName, {?MODULE, Type, [Async, PreparedKey, Data]}) of {error, Reason} = Result -> ?tp( error, @@ -292,7 +292,7 @@ exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when end. exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> - case ecpool:pick_and_do(PoolName, {?MODULE, batch_query, [Async, CQLs]}, no_handover) of + case exec(PoolName, {?MODULE, batch_query, [Async, CQLs]}) of {error, Reason} = Result -> ?tp( error, @@ -305,6 +305,13 @@ exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> Result end. +%% Pick one of the pool members to do the query. +%% Using 'no_handoever' strategy, +%% meaning the buffer worker does the gen_server call or gen_server cast +%% towards the connection process. +exec(PoolName, Query) -> + ecpool:pick_and_do(PoolName, Query, no_handover). + on_get_status(_InstId, #{poolname := Pool} = State) -> case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of true -> @@ -343,17 +350,23 @@ do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepare query(Conn, sync, CQL, Params) -> ecql:query(Conn, CQL, Params); query(Conn, {async, Callback}, CQL, Params) -> - ecql:async_query(Conn, CQL, Params, one, Callback). + ok = ecql:async_query(Conn, CQL, Params, one, Callback), + %% return the connection pid for buffer worker to monitor + {ok, Conn}. prepared_query(Conn, sync, PreparedKey, Params) -> ecql:execute(Conn, PreparedKey, Params); prepared_query(Conn, {async, Callback}, PreparedKey, Params) -> - ecql:async_execute(Conn, PreparedKey, Params, Callback). + ok = ecql:async_execute(Conn, PreparedKey, Params, Callback), + %% return the connection pid for buffer worker to monitor + {ok, Conn}. batch_query(Conn, sync, Rows) -> ecql:batch(Conn, Rows); batch_query(Conn, {async, Callback}, Rows) -> - ecql:async_batch(Conn, Rows, Callback). + ok = ecql:async_batch(Conn, Rows, Callback), + %% return the connection pid for buffer worker to monitor + {ok, Conn}. %%-------------------------------------------------------------------- %% callbacks for ecpool From 28a68a0ec78cebe4b6e7ddc3cd72fd105982963f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 11:40:27 +0200 Subject: [PATCH 098/194] refactor: stop i18n support in hotconf and bridges frontend team has decided to deal with translations all by themselves --- apps/emqx_conf/src/emqx_conf.erl | 64 ++++------------- .../src/emqx_dashboard_schema_api.erl | 24 +++---- .../src/emqx_dashboard_swagger.erl | 68 ++++++++++++++++++- 3 files changed, 88 insertions(+), 68 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 8632df139..8d67cfb57 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -31,8 +31,9 @@ %% TODO: move to emqx_dashboard when we stop building api schema at build time -export([ - hotconf_schema_json/1, - bridge_schema_json/1 + hotconf_schema_json/0, + bridge_schema_json/0, + hocon_schema_to_spec/2 ]). %% for rpc @@ -184,13 +185,13 @@ gen_api_schema_json(Dir, Lang) -> %% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_hotconf(Dir, Lang) -> File = schema_filename(Dir, "hot-config-schema-", Lang), - IoData = hotconf_schema_json(Lang), + IoData = hotconf_schema_json(), ok = write_api_schema_json_file(File, IoData). %% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_bridge(Dir, Lang) -> File = schema_filename(Dir, "bridge-api-", Lang), - IoData = bridge_schema_json(Lang), + IoData = bridge_schema_json(), ok = write_api_schema_json_file(File, IoData). %% TODO: delete this function when we stop generating this JSON at build time. @@ -199,14 +200,14 @@ write_api_schema_json_file(File, IoData) -> file:write_file(File, IoData). %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. -hotconf_schema_json(Lang) -> +hotconf_schema_json() -> SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, - gen_api_schema_json_iodata(emqx_mgmt_api_configs, SchemaInfo, Lang). + gen_api_schema_json_iodata(emqx_mgmt_api_configs, SchemaInfo). %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. -bridge_schema_json(Lang) -> +bridge_schema_json() -> SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, - gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo, Lang). + gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo). schema_filename(Dir, Prefix, Lang) -> Filename = Prefix ++ Lang ++ ".json", @@ -270,50 +271,11 @@ gen_example(File, SchemaModule) -> Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). -%% TODO: move this to emqx_dashboard when we stop generating -%% this JSON at build time. -gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Lang) -> - {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( +gen_api_schema_json_iodata(SchemaMod, SchemaInfo) -> + emqx_dashboard_swagger:gen_api_schema_json_iodata( SchemaMod, - #{ - schema_converter => fun hocon_schema_to_spec/2, - i18n_lang => Lang - } - ), - ApiSpec = lists:foldl( - fun({Path, Spec, _, _}, Acc) -> - NewSpec = maps:fold( - fun(Method, #{responses := Responses}, SubAcc) -> - case Responses of - #{ - <<"200">> := - #{ - <<"content">> := #{ - <<"application/json">> := #{<<"schema">> := Schema} - } - } - } -> - SubAcc#{Method => Schema}; - _ -> - SubAcc - end - end, - #{}, - Spec - ), - Acc#{list_to_atom(Path) => NewSpec} - end, - #{}, - ApiSpec0 - ), - Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - emqx_utils_json:encode( - #{ - info => SchemaInfo, - paths => ApiSpec, - components => #{schemas => Components} - }, - [pretty, force_utf8] + SchemaInfo, + fun ?MODULE:hocon_schema_to_spec/2 ). -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl index 898d95b3c..e4f2f0c1a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl @@ -45,18 +45,11 @@ schema("/schemas/:name") -> 'operationId' => get_schema, get => #{ parameters => [ - {name, hoconsc:mk(hoconsc:enum([hotconf, bridges]), #{in => path})}, - {lang, - hoconsc:mk(typerefl:string(), #{ - in => query, - default => <<"en">>, - desc => <<"The language of the schema.">> - })} + {name, hoconsc:mk(hoconsc:enum([hotconf, bridges]), #{in => path})} ], desc => << "Get the schema JSON of the specified name. " - "NOTE: you should never need to make use of this API " - "unless you are building a multi-lang dashboaard." + "NOTE: only intended for EMQX Dashboard." >>, tags => ?TAGS, security => [], @@ -71,14 +64,13 @@ schema("/schemas/:name") -> %%-------------------------------------------------------------------- get_schema(get, #{ - bindings := #{name := Name}, - query_string := #{<<"lang">> := Lang} + bindings := #{name := Name} }) -> - {200, gen_schema(Name, iolist_to_binary(Lang))}; + {200, gen_schema(Name)}; get_schema(get, _) -> {400, ?BAD_REQUEST, <<"unknown">>}. -gen_schema(hotconf, Lang) -> - emqx_conf:hotconf_schema_json(Lang); -gen_schema(bridges, Lang) -> - emqx_conf:bridge_schema_json(Lang). +gen_schema(hotconf) -> + emqx_conf:hotconf_schema_json(); +gen_schema(bridges) -> + emqx_conf:bridge_schema_json(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e471486e5..fec9717ba 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -26,7 +26,11 @@ -export([error_codes/1, error_codes/2]). -export([file_schema/1]). --export([filter_check_request/2, filter_check_request_and_translate_body/2]). +-export([ + filter_check_request/2, + filter_check_request_and_translate_body/2, + gen_api_schema_json_iodata/3 +]). -ifdef(TEST). -export([ @@ -72,6 +76,8 @@ ]) ). +-define(SPECIAL_LANG_MSGID, <<"$msgid">>). + -define(MAX_ROW_LIMIT, 1000). -define(DEFAULT_ROW, 100). @@ -192,6 +198,50 @@ file_schema(FileName) -> } }. +gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Converter) -> + {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( + SchemaMod, + #{ + schema_converter => Converter, + i18n_lang => ?SPECIAL_LANG_MSGID + } + ), + ApiSpec = lists:foldl( + fun({Path, Spec, _, _}, Acc) -> + NewSpec = maps:fold( + fun(Method, #{responses := Responses}, SubAcc) -> + case Responses of + #{ + <<"200">> := + #{ + <<"content">> := #{ + <<"application/json">> := #{<<"schema">> := Schema} + } + } + } -> + SubAcc#{Method => Schema}; + _ -> + SubAcc + end + end, + #{}, + Spec + ), + Acc#{list_to_atom(Path) => NewSpec} + end, + #{}, + ApiSpec0 + ), + Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), + emqx_utils_json:encode( + #{ + info => SchemaInfo, + paths => ApiSpec, + components => #{schemas => Components} + }, + [pretty, force_utf8] + ). + %%------------------------------------------------------------------------------ %% Private functions %%------------------------------------------------------------------------------ @@ -482,6 +532,14 @@ maybe_add_summary_from_label(Spec, Hocon, Options) -> get_i18n(Tag, ?DESC(Namespace, Id), Default, Options) -> Lang = get_lang(Options), + case Lang of + ?SPECIAL_LANG_MSGID -> + make_msgid(Namespace, Id, Tag); + _ -> + get_i18n_text(Lang, Namespace, Id, Tag, Default) + end. + +get_i18n_text(Lang, Namespace, Id, Tag, Default) -> case emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, Tag) of undefined -> Default; @@ -489,6 +547,14 @@ get_i18n(Tag, ?DESC(Namespace, Id), Default, Options) -> Text end. +%% Format：$msgid:Namespace.Id.Tag +%% e.g. $msgid:emqx_schema.key.desc +%% $msgid:emqx_schema.key.label +%% if needed, the consumer of this schema JSON can use this msgid to +%% resolve the text in the i18n database. +make_msgid(Namespace, Id, Tag) -> + iolist_to_binary(["$msgid:", to_bin(Namespace), ".", to_bin(Id), ".", Tag]). + %% So far i18n_lang in options is only used at build time. %% At runtime, it's still the global config which controls the language. get_lang(#{i18n_lang := Lang}) -> Lang; From 55c488fa95262663da6475779bd15bb6e2a6cd07 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 13:42:42 +0200 Subject: [PATCH 099/194] refactor: stop generating static hot-conf and bridges schema files --- apps/emqx_conf/src/emqx_conf.erl | 27 --------------------------- build | 7 ++----- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 8d67cfb57..eaa16ab5a 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -150,7 +150,6 @@ dump_schema(Dir, SchemaModule) -> lists:foreach( fun(Lang) -> ok = gen_config_md(Dir, SchemaModule, Lang), - ok = gen_api_schema_json(Dir, Lang), ok = gen_schema_json(Dir, SchemaModule, Lang) end, ["en", "zh"] @@ -177,28 +176,6 @@ gen_schema_json(Dir, SchemaModule, Lang) -> IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). -%% TODO: delete this function when we stop generating this JSON at build time. -gen_api_schema_json(Dir, Lang) -> - gen_api_schema_json_hotconf(Dir, Lang), - gen_api_schema_json_bridge(Dir, Lang). - -%% TODO: delete this function when we stop generating this JSON at build time. -gen_api_schema_json_hotconf(Dir, Lang) -> - File = schema_filename(Dir, "hot-config-schema-", Lang), - IoData = hotconf_schema_json(), - ok = write_api_schema_json_file(File, IoData). - -%% TODO: delete this function when we stop generating this JSON at build time. -gen_api_schema_json_bridge(Dir, Lang) -> - File = schema_filename(Dir, "bridge-api-", Lang), - IoData = bridge_schema_json(), - ok = write_api_schema_json_file(File, IoData). - -%% TODO: delete this function when we stop generating this JSON at build time. -write_api_schema_json_file(File, IoData) -> - io:format(user, "===< Generating: ~s~n", [File]), - file:write_file(File, IoData). - %% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. hotconf_schema_json() -> SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, @@ -209,10 +186,6 @@ bridge_schema_json() -> SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo). -schema_filename(Dir, Prefix, Lang) -> - Filename = Prefix ++ Lang ++ ".json", - filename:join([Dir, Filename]). - %% TODO: remove it and also remove hocon_md.erl and friends. %% markdown generation from schema is a failure and we are moving to an interactive %% viewer like swagger UI. diff --git a/build b/build index 77d4dbfc8..05246a359 100755 --- a/build +++ b/build @@ -92,7 +92,7 @@ log() { } make_docs() { - local libs_dir1 libs_dir2 libs_dir3 docdir dashboard_www_static + local libs_dir1 libs_dir2 libs_dir3 docdir libs_dir1="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" if [ -d "_build/default/lib/" ]; then libs_dir2="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)" @@ -113,14 +113,11 @@ make_docs() { ;; esac docdir="_build/docgen/$PROFILE" - dashboard_www_static='apps/emqx_dashboard/priv/www/static/' - mkdir -p "$docdir" "$dashboard_www_static" + mkdir -p "$docdir" # shellcheck disable=SC2086 erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \ "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." - cp "$docdir"/bridge-api-*.json "$dashboard_www_static" - cp "$docdir"/hot-config-schema-*.json "$dashboard_www_static" } assert_no_compile_time_only_deps() { From 9260b5ec6c55fc8ae6deb80c65a328216a745a79 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 19:13:55 +0200 Subject: [PATCH 100/194] test(emqx_dashboard): add test case for api/v5/schemas API --- .../test/emqx_dashboard_schema_api_SUITE.erl | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 apps/emqx_dashboard/test/emqx_dashboard_schema_api_SUITE.erl diff --git a/apps/emqx_dashboard/test/emqx_dashboard_schema_api_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_schema_api_SUITE.erl new file mode 100644 index 000000000..e4425aed8 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_schema_api_SUITE.erl @@ -0,0 +1,52 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_dashboard_schema_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx/include/http_api.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-define(SERVER, "http://127.0.0.1:18083/api/v5"). + +-import(emqx_mgmt_api_test_util, [request/2]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf]), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_conf]). + +t_hotconf(_) -> + Url = ?SERVER ++ "/schemas/hotconf", + {ok, 200, Body} = request(get, Url), + %% assert it's a valid json + _ = emqx_utils_json:decode(Body), + ok. + +t_bridges(_) -> + Url = ?SERVER ++ "/schemas/bridges", + {ok, 200, Body} = request(get, Url), + %% assert it's a valid json + _ = emqx_utils_json:decode(Body), + ok. From ed7a8659d2022c50f1f33b86cd49eb271fdab433 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 22:22:03 +0200 Subject: [PATCH 101/194] feat: add a json format support for the /status API --- .../src/emqx_mgmt_api_status.erl | 59 ++++++++++++-- .../test/emqx_mgmt_api_status_SUITE.erl | 79 ++++++++++++++++++- rel/i18n/emqx_mgmt_api_status.hocon | 31 ++++++-- rel/i18n/zh/emqx_mgmt_api_status.hocon | 22 ++++-- 4 files changed, 171 insertions(+), 20 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl index 7d5c18e59..c0ee42e2b 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_status.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl @@ -45,6 +45,17 @@ schema("/status") -> #{ 'operationId' => get_status, get => #{ + parameters => [ + {format, + hoconsc:mk( + string(), + #{ + in => query, + default => <<"text">>, + desc => ?DESC(get_status_api_format) + } + )} + ], description => ?DESC(get_status_api), tags => ?TAGS, security => [], @@ -70,7 +81,16 @@ path() -> "/status". init(Req0, State) -> - {Code, Headers, Body} = running_status(), + Format = + try + QS = cowboy_req:parse_qs(Req0), + {_, F} = lists:keyfind(<<"format">>, 1, QS), + F + catch + _:_ -> + <<"text">> + end, + {Code, Headers, Body} = running_status(Format), Req = cowboy_req:reply(Code, Headers, Body, Req0), {ok, Req, State}. @@ -78,29 +98,52 @@ init(Req0, State) -> %% API Handler funcs %%-------------------------------------------------------------------- -get_status(get, _Params) -> - running_status(). +get_status(get, Params) -> + Format = maps:get(<<"format">>, maps:get(query_string, Params, #{}), <<"text">>), + running_status(iolist_to_binary(Format)). -running_status() -> +running_status(Format) -> case emqx_dashboard_listener:is_ready(timer:seconds(20)) of true -> - BrokerStatus = broker_status(), AppStatus = application_status(), - Body = io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), BrokerStatus, AppStatus]), + Body = do_get_status(AppStatus, Format), StatusCode = case AppStatus of running -> 200; not_running -> 503 end, + ContentType = + case Format of + <<"json">> -> <<"applicatin/json">>; + _ -> <<"text/plain">> + end, Headers = #{ - <<"content-type">> => <<"text/plain">>, + <<"content-type">> => ContentType, <<"retry-after">> => <<"15">> }, - {StatusCode, Headers, list_to_binary(Body)}; + {StatusCode, Headers, iolist_to_binary(Body)}; false -> {503, #{<<"retry-after">> => <<"15">>}, <<>>} end. +do_get_status(AppStatus, <<"json">>) -> + BrokerStatus = broker_status(), + emqx_utils_json:encode(#{ + node_name => atom_to_binary(node(), utf8), + rel_vsn => vsn(), + broker_status => atom_to_binary(BrokerStatus), + app_status => atom_to_binary(AppStatus) + }); +do_get_status(AppStatus, _) -> + BrokerStatus = broker_status(), + io_lib:format("Node ~ts is ~ts~nemqx is ~ts", [node(), BrokerStatus, AppStatus]). + +vsn() -> + iolist_to_binary([ + emqx_release:edition_vsn_prefix(), + emqx_release:version() + ]). + broker_status() -> case emqx:is_running() of true -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl index f0200c410..e8e0b4ac9 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_status_SUITE.erl @@ -38,7 +38,10 @@ all() -> get_status_tests() -> [ t_status_ok, - t_status_not_ok + t_status_not_ok, + t_status_text_format, + t_status_json_format, + t_status_bad_format_qs ]. groups() -> @@ -87,8 +90,10 @@ do_request(Opts) -> headers := Headers, body := Body0 } = Opts, + QS = maps:get(qs, Opts, ""), URL = ?HOST ++ filename:join(Path0), - {ok, #{host := Host, port := Port, path := Path}} = emqx_http_lib:uri_parse(URL), + {ok, #{host := Host, port := Port, path := Path1}} = emqx_http_lib:uri_parse(URL), + Path = Path1 ++ QS, %% we must not use `httpc' here, because it keeps retrying when it %% receives a 503 with `retry-after' header, and there's no option %% to stop that behavior... @@ -165,3 +170,73 @@ t_status_not_ok(Config) -> Headers ), ok. + +t_status_text_format(Config) -> + Path = ?config(get_status_path, Config), + #{ + body := Resp, + status_code := StatusCode + } = do_request(#{ + method => get, + path => Path, + qs => "?format=text", + headers => [], + body => no_body + }), + ?assertEqual(200, StatusCode), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is running$">>) + ), + ok. + +t_status_json_format(Config) -> + Path = ?config(get_status_path, Config), + #{ + body := Resp, + status_code := StatusCode + } = do_request(#{ + method => get, + path => Path, + qs => "?format=json", + headers => [], + body => no_body + }), + ?assertEqual(200, StatusCode), + ?assertMatch( + #{<<"app_status">> := <<"running">>}, + emqx_utils_json:decode(Resp) + ), + ok. + +t_status_bad_format_qs(Config) -> + lists:foreach( + fun(QS) -> + test_status_bad_format_qs(QS, Config) + end, + [ + "?a=b", + "?format=", + "?format=x" + ] + ). + +%% when query-sting is invalid, fallback to text format +test_status_bad_format_qs(QS, Config) -> + Path = ?config(get_status_path, Config), + #{ + body := Resp, + status_code := StatusCode + } = do_request(#{ + method => get, + path => Path, + qs => QS, + headers => [], + body => no_body + }), + ?assertEqual(200, StatusCode), + ?assertMatch( + {match, _}, + re:run(Resp, <<"emqx is running$">>) + ), + ok. diff --git a/rel/i18n/emqx_mgmt_api_status.hocon b/rel/i18n/emqx_mgmt_api_status.hocon index 28278b747..2034d13bc 100644 --- a/rel/i18n/emqx_mgmt_api_status.hocon +++ b/rel/i18n/emqx_mgmt_api_status.hocon @@ -1,21 +1,42 @@ emqx_mgmt_api_status { get_status_api.desc: -"""Serves as a health check for the node. Returns a plain text response describing the status of the node. This endpoint requires no authentication. +"""Serves as a health check for the node. +Returns response to describe the status of the node and the application. + +This endpoint requires no authentication. Returns status code 200 if the EMQX application is up and running, 503 otherwise. This API was introduced in v5.0.10. -The GET `/status` endpoint (without the `/api/...` prefix) is also an alias to this endpoint and works in the same way. This alias has been available since v5.0.0.""" +The GET `/status` endpoint (without the `/api/...` prefix) is also an alias to this endpoint and works in the same way. +This alias has been available since v5.0.0. + +Starting from v5.0.25 or e5.0.4, you can also use 'format' parameter to get JSON format information. +""" get_status_api.label: """Service health check""" get_status_response200.desc: -"""Node emqx@127.0.0.1 is started +"""If 'format' parameter is 'json', then it returns a JSON like below:
+{ + "rel_vsn": "v5.0.23", + "node_name": "emqx@127.0.0.1", + "broker_status": "started", + "app_status": "running" +} +
+Otherwise it returns free text strings as below:
+Node emqx@127.0.0.1 is started emqx is running""" get_status_response503.desc: -"""Node emqx@127.0.0.1 is stopped -emqx is not_running""" +"""When EMQX application is temporary not running or being restarted, it may return 'emqx is not_running'. +If the 'format' parameter is provided 'json', the nthe 'app_status' field in the JSON object is 'not_running'. +""" + +get_status_api_format.desc: +"""Specify the response format, 'text' (default) to return the HTTP body in free text, +or 'json' to return the HTTP body with a JSON object.""" } diff --git a/rel/i18n/zh/emqx_mgmt_api_status.hocon b/rel/i18n/zh/emqx_mgmt_api_status.hocon index 3625db967..3938f47c1 100644 --- a/rel/i18n/zh/emqx_mgmt_api_status.hocon +++ b/rel/i18n/zh/emqx_mgmt_api_status.hocon @@ -1,22 +1,34 @@ emqx_mgmt_api_status { get_status_api.desc: -"""作为节点的健康检查。 返回一个纯文本的响应，描述节点的状态。 +"""节点的健康检查。 返回节点状态的描述信息。 如果 EMQX 应用程序已经启动并运行，返回状态代码 200，否则返回 503。 这个API是在v5.0.10中引入的。 -GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。 这个别名从v5.0.0开始就有了。""" +GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。 这个别名从v5.0.0开始就有了。 +自 v5.0.25 和 e5.0.4 开始，可以通过指定 'format' 参数来得到 JSON 格式的信息。""" get_status_api.label: """服务健康检查""" get_status_response200.desc: -"""Node emqx@127.0.0.1 is started +"""如果 'format' 参数为 'json'，则返回如下JSON：
+{ + "rel_vsn": "v5.0.23", + "node_name": "emqx@127.0.0.1", + "broker_status": "started", + "app_status": "running" +} +
+否则返回2行自由格式的文本，第一行描述节点的状态，第二行描述 EMQX 应用运行状态。例如：
+Node emqx@127.0.0.1 is started emqx is running""" get_status_response503.desc: -"""Node emqx@127.0.0.1 is stopped -emqx is not_running""" +"""如果 EMQX 应用暂时没有启动，或正在重启，则可能返回 'emqx is not_running'""" + +get_status_api_format.desc: +"""指定返回的内容格式。使用 'text'（默认）则返回自由格式的字符串； 'json' 则返回 JSON 格式。""" } From 48e68b7c77ced9c1a243fbd09ddde8e9065c0cf9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 18:04:07 +0200 Subject: [PATCH 102/194] test: add smoke test to cover schemas api --- scripts/test/emqx-smoke-test.sh | 102 ++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 26 deletions(-) diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index ce8116b39..09e7d6438 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -2,42 +2,92 @@ set -euo pipefail -[ $# -ne 2 ] && { echo "Usage: $0 ip port"; exit 1; } +[ $# -ne 2 ] && { echo "Usage: $0 host port"; exit 1; } -IP=$1 +HOST=$1 PORT=$2 -URL="http://$IP:$PORT/status" +BASE_URL="http://$HOST:$PORT" ## Check if EMQX is responding -ATTEMPTS=10 -while ! curl "$URL" >/dev/null 2>&1; do - if [ $ATTEMPTS -eq 0 ]; then - echo "emqx is not responding on $URL" - exit 1 +wait_for_emqx() { + local attempts=10 + local url="$BASE_URL"/status + while ! curl "$url" >/dev/null 2>&1; do + if [ $attempts -eq 0 ]; then + echo "emqx is not responding on $url" + exit 1 + fi + sleep 5 + attempts=$((attempts-1)) + done +} + +## Get the JSON format status which is jq friendly and includes a version string +json_status() { + local url="${BASE_URL}/status?format=json" + local resp + resp="$(curl -s "$url")" + if (echo "$resp" | jq . >/dev/null 2>&1); then + echo "$resp" + else + echo 'NOT_JSON' fi - sleep 5 - ATTEMPTS=$((ATTEMPTS-1)) -done +} ## Check if the API docs are available -API_DOCS_URL="http://$IP:$PORT/api-docs/index.html" -API_DOCS_STATUS="$(curl -s -o /dev/null -w "%{http_code}" "$API_DOCS_URL")" -if [ "$API_DOCS_STATUS" != "200" ]; then - echo "emqx is not responding on $API_DOCS_URL" - exit 1 -fi +check_api_docs() { + local url="$BASE_URL/api-docs/index.html" + local status + status="$(curl -s -o /dev/null -w "%{http_code}" "$url")" + if [ "$status" != "200" ]; then + echo "emqx is not responding on $API_DOCS_URL" + exit 1 + fi +} ## Check if the swagger.json contains hidden fields ## fail if it does -SWAGGER_JSON_URL="http://$IP:$PORT/api-docs/swagger.json" -## assert swagger.json is valid json -JSON="$(curl -s "$SWAGGER_JSON_URL")" -echo "$JSON" | jq . >/dev/null +check_swagger_json() { + local url="$BASE_URL/api-docs/swagger.json" + ## assert swagger.json is valid json + JSON="$(curl -s "$url")" + echo "$JSON" | jq . >/dev/null -if [ "${EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS:-yes}" = 'yes' ]; then - ## assert swagger.json does not contain trie_compaction (which is a hidden field) - if echo "$JSON" | grep -q trie_compaction; then - echo "swagger.json contains hidden fields" + if [ "${EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS:-yes}" = 'yes' ]; then + ## assert swagger.json does not contain trie_compaction (which is a hidden field) + if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 + fi + fi +} + +check_schema_json() { + local name="$1" + local expected_title="$2" + local url="$BASE_URL/api/v5/schemas/$name" + local json + json="$(curl -s "$url" | jq .)" + title="$(echo "$json" | jq -r '.info.title')" + if [[ "$title" != "$expected_title" ]]; then + echo "unexpected value from GET $url" + echo "expected: $expected_title" + echo "got : $title" exit 1 fi -fi +} + +main() { + wait_for_emqx + local JSON_STATUS + JSON_STATUS="$(json_status)" + check_api_docs + check_swagger_json + ## The json status feature was added after hotconf and bridges schema API + if [ "$JSON_STATUS" != 'NOT_JSON' ]; then + check_schema_json hotconf "EMQX Hot Conf API Schema" + check_schema_json bridges "EMQX Data Bridge API Schema" + fi +} + +main From 0bd30e039f300eb049009325d122895d356cbab9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 25 Apr 2023 22:56:25 +0200 Subject: [PATCH 103/194] test: simplify swagger json check script --- .github/workflows/build_slim_packages.yaml | 3 --- scripts/test/emqx-smoke-test.sh | 13 +++++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 9ae5ba944..06bcb98a2 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -194,15 +194,12 @@ jobs: run: | CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) - export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='yes' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT docker stop $CID - name: test two nodes cluster with proto_dist=inet_tls in docker run: | ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy) - # versions before 5.0.22 have hidden fields included in the API spec - export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='no' ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT # cleanup ./scripts/test/start-two-nodes-in-docker.sh -c diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh index 09e7d6438..44df5b5bd 100755 --- a/scripts/test/emqx-smoke-test.sh +++ b/scripts/test/emqx-smoke-test.sh @@ -52,13 +52,10 @@ check_swagger_json() { ## assert swagger.json is valid json JSON="$(curl -s "$url")" echo "$JSON" | jq . >/dev/null - - if [ "${EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS:-yes}" = 'yes' ]; then - ## assert swagger.json does not contain trie_compaction (which is a hidden field) - if echo "$JSON" | grep -q trie_compaction; then - echo "swagger.json contains hidden fields" - exit 1 - fi + ## assert swagger.json does not contain trie_compaction (which is a hidden field) + if echo "$JSON" | grep -q trie_compaction; then + echo "swagger.json contains hidden fields" + exit 1 fi } @@ -82,9 +79,9 @@ main() { local JSON_STATUS JSON_STATUS="$(json_status)" check_api_docs - check_swagger_json ## The json status feature was added after hotconf and bridges schema API if [ "$JSON_STATUS" != 'NOT_JSON' ]; then + check_swagger_json check_schema_json hotconf "EMQX Hot Conf API Schema" check_schema_json bridges "EMQX Data Bridge API Schema" fi From 01770fab8597d9eda6afcb8fe7704206a4e56c14 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 26 Apr 2023 18:02:14 +0200 Subject: [PATCH 104/194] ci: fix pkg-vsn.sh in perf test --- .github/workflows/performance_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index e8a2a321e..2433f3621 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -24,14 +24,14 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event.inputs.ref }} + - name: Work around https://github.com/actions/checkout/issues/766 + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" - id: prepare run: | echo "EMQX_NAME=emqx" >> $GITHUB_ENV echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT - - name: Work around https://github.com/actions/checkout/issues/766 - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Build deb package run: | make ${EMQX_NAME}-pkg From 35c48ef009975201488373ba51f3cc9acb24ae16 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 26 Apr 2023 18:02:44 +0200 Subject: [PATCH 105/194] chore: v5.0.24 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index eff228621..0cfba8fe3 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.23"). +-define(EMQX_RELEASE_CE, "5.0.24"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.3-alpha.3"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 312a9dfbe..9c23f7c15 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.23 +version: 5.0.24 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.23 +appVersion: 5.0.24 From 50504a4cbfcb29c48908421876a92d09194e3e9f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 26 Apr 2023 18:03:00 +0200 Subject: [PATCH 106/194] docs: Generate changelog for v5.0.24 --- changes/v5.0.24.en.md | 89 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 changes/v5.0.24.en.md diff --git a/changes/v5.0.24.en.md b/changes/v5.0.24.en.md new file mode 100644 index 000000000..4fa5cdd4f --- /dev/null +++ b/changes/v5.0.24.en.md @@ -0,0 +1,89 @@ +# v5.0.24 + +## Enhancements + +- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecates the integration with StatsD. + + There seemd to be no user using StatsD integration, so we have decided to hide this feature + for now. We will either remove or revive it based on requirements in the future. + +- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low level, + in most cases, users only need to manage plugins on the dashboard + without the need for manual modification, so we lowered the level. + +- [#10491](https://github.com/emqx/emqx/pull/10491) Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file. + +- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files, + Now we can store Unicode characters normally. + For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\"" + +- [#10487](https://github.com/emqx/emqx/pull/10487) Optimize the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage. + +- [#10490](https://github.com/emqx/emqx/pull/10490) Remove the default limit of connect rate which used to be `1000/s` + +## Bug Fixes + +- [#10407](https://github.com/emqx/emqx/pull/10407) Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding + unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated. + Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager + doesn't crash because of alarm timeouts. + The crashes were possible when the following conditions co-occurred: + - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors; + - the system experienced a very high load. + +- [#10420](https://github.com/emqx/emqx/pull/10420) Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. + * Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). + * Fix the issue that path segments could be HTTP encoded twice. + +- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. + +- [#10448](https://github.com/emqx/emqx/pull/10448) Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. + + In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. + +- [#10449](https://github.com/emqx/emqx/pull/10449) Validate the ssl_options and header configurations when creating authentication http (`authn_http`). + Prior to this, incorrect `ssl` configuration could result in successful creation but the entire authn being unusable. + +- [#10455](https://github.com/emqx/emqx/pull/10455) Fixed an issue that could cause (otherwise harmless) noise in the logs. + + During some particularly slow synchronous calls to bridges, some late replies could be sent to connections processes that were no longer expecting a reply, and then emit an error log like: + + ``` + 2023-04-19T18:24:35.350233+00:00 [error] msg: unexpected_info, mfa: emqx_channel:handle_info/2, line: 1278, peername: 172.22.0.1:36384, clientid: caribdis_bench_sub_1137967633_4788, info: {#Ref<0.408802983.1941504010.189402>,{ok,200,[{<<"cache-control">>,<<"max-age=0, ...">>}} + ``` + + Those logs are harmless, but they could flood and worry the users without need. + +- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecate config `broker.shared_dispatch_ack_enabled`. + This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected. + However since v5.0.9, this feature is no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group. + See also: https://github.com/emqx/emqx/pull/9104 + +- [#10463](https://github.com/emqx/emqx/pull/10463) Improve bridges API error handling. + If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'. + +- [#10484](https://github.com/emqx/emqx/pull/10484) Fix the issue that the priority of the configuration cannot be set during rolling upgrade. + For example, when authorization is modified in v5.0.21 and then upgraded v5.0.23 through rolling upgrade, + the authorization will be restored to the default. + +- [#10495](https://github.com/emqx/emqx/pull/10495) Add the limiter API `/configs/limiter` which was deleted by mistake back. + +- [#10500](https://github.com/emqx/emqx/pull/10500) Add several fixes, enhancements and features in Mria: + - protect `mria:join/1,2` with a global lock to prevent conflicts between + two nodes trying to join each other simultaneously + [Mria PR](https://github.com/emqx/mria/pull/137) + - implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until + a transaction is imported to the local node (if the local node is a replicant, otherwise, + it behaves exactly the same as `mria:transaction/3,2`) + [Mria PR](https://github.com/emqx/mria/pull/136) + - optimize `mria:running_nodes/0` + [Mria PR](https://github.com/emqx/mria/pull/135) + - optimize `mria:ro_transaction/2` when called on a replicant node + [Mria PR](https://github.com/emqx/mria/pull/134). + +- [#10518](https://github.com/emqx/emqx/pull/10518) Add the following fixes and features in Mria: + - call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership + gen_server won't crash if RPC to another node fails + [Mria PR](https://github.com/emqx/mria/pull/139) + - Add extra field to ?rlog_sync table to facilitate extending this functionality in future + [Mria PR](https://github.com/emqx/mria/pull/138). From c53741a08c09d3b43b58a6d7635cebf2bcb30697 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 17:42:27 -0300 Subject: [PATCH 107/194] fix(buffer_worker): avoid sending late reply messages to callers Fixes https://emqx.atlassian.net/browse/EMQX-9635 During a sync call from process `A` to a buffer worker `B`, its call to the underlying resource `C` can be very slow. In those cases, `A` will receive a timeout response and expect no more messages from `B` nor `C`. However, prior to this fix, if `B` is stuck in a long sync call to `C` and then gets its response after `A` timed out, `B` would still send the late response to `A`, polluting its mailbox. --- .../src/emqx_resource_buffer_worker.erl | 17 +++++-- .../test/emqx_connector_demo.erl | 6 ++- .../test/emqx_resource_SUITE.erl | 45 +++++++++++++++++++ changes/ce/fix-10455.en.md | 9 ++++ 4 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 changes/ce/fix-10455.en.md diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index e34cf5d0a..2e2cd5631 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -52,7 +52,7 @@ -export([queue_item_marshaller/1, estimate_size/1]). --export([handle_async_reply/2, handle_async_batch_reply/2]). +-export([handle_async_reply/2, handle_async_batch_reply/2, reply_call/2]). -export([clear_disk_queue_dir/2]). @@ -293,10 +293,8 @@ code_change(_OldVsn, State, _Extra) -> pick_call(Id, Key, Query, Timeout) -> ?PICK(Id, Key, Pid, begin - Caller = self(), MRef = erlang:monitor(process, Pid, [{alias, reply_demonitor}]), - From = {Caller, MRef}, - ReplyTo = {fun gen_statem:reply/2, [From]}, + ReplyTo = {fun ?MODULE:reply_call/2, [MRef]}, erlang:send(Pid, ?SEND_REQ(ReplyTo, Query)), receive {MRef, Response} -> @@ -1703,6 +1701,17 @@ default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) -> default_resume_interval(RequestTimeout, HealthCheckInterval) -> max(1, min(HealthCheckInterval, RequestTimeout div 3)). +-spec reply_call(reference(), term()) -> ok. +reply_call(Alias, Response) -> + %% Since we use a reference created with `{alias, + %% reply_demonitor}', after we `demonitor' it in case of a + %% timeout, we won't send any more messages that the caller is not + %% expecting anymore. Using `gen_statem:reply({pid(), + %% reference()}, _)' would still send a late reply even after the + %% demonitor. + erlang:send(Alias, {Alias, Response}), + ok. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). adjust_batch_time_test_() -> diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index a1393c574..5be854e93 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -144,7 +144,11 @@ on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) -> Result after 1000 -> {error, timeout} - end. + end; +on_query(_InstId, {sync_sleep_before_reply, SleepFor}, _State) -> + %% This simulates a slow sync call + timer:sleep(SleepFor), + {ok, slept}. on_query_async(_InstId, block, ReplyFun, #{pid := Pid}) -> Pid ! {block, ReplyFun}, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 385b4cb91..e098c2e1c 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2751,6 +2751,51 @@ t_volatile_offload_mode(_Config) -> end ). +t_late_call_reply(_Config) -> + emqx_connector_demo:set_callback_mode(always_sync), + RequestTimeout = 500, + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + buffer_mode => memory_only, + request_timeout => RequestTimeout, + query_mode => sync + } + ) + ), + ?check_trace( + begin + %% Sleep for longer than the request timeout; the call reply will + %% have been already returned (a timeout), but the resource will + %% still send a message with the reply. + %% The demo connector will reply with `{error, timeout}' after 1 s. + SleepFor = RequestTimeout + 500, + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + emqx_resource:query( + ?ID, + {sync_sleep_before_reply, SleepFor}, + #{timeout => RequestTimeout} + ) + ), + %% Our process shouldn't receive any late messages. + receive + LateReply -> + ct:fail("received late reply: ~p", [LateReply]) + after SleepFor -> + ok + end, + ok + end, + [] + ), + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-10455.en.md b/changes/ce/fix-10455.en.md new file mode 100644 index 000000000..07d8c71db --- /dev/null +++ b/changes/ce/fix-10455.en.md @@ -0,0 +1,9 @@ +Fixed an issue that could cause (otherwise harmless) noise in the logs. + +During some particularly slow synchronous calls to bridges, some late replies could be sent to connections processes that were no longer expecting a reply, and then emit an error log like: + +``` +2023-04-19T18:24:35.350233+00:00 [error] msg: unexpected_info, mfa: emqx_channel:handle_info/2, line: 1278, peername: 172.22.0.1:36384, clientid: caribdis_bench_sub_1137967633_4788, info: {#Ref<0.408802983.1941504010.189402>,{ok,200,[{<<"cache-control">>,<<"max-age=0, ...">>}} +``` + +Those logs are harmless, but they could flood and worry the users without need. From 7967090de0f71c5920f0e1fa3954403d65d00ff0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Apr 2023 20:09:33 +0200 Subject: [PATCH 108/194] chore(emqx_dashboard): ignore everything in priv dir --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 62e8ddc81..6c4de9272 100644 --- a/.gitignore +++ b/.gitignore @@ -43,8 +43,7 @@ tmp/ _packages elvis emqx_dialyzer_*_plt -*/emqx_dashboard/priv/www -*/emqx_dashboard/priv/i18n.conf +*/emqx_dashboard/priv/ dist.zip scripts/git-token apps/*/etc/*.all From a8b000f06291a80e999b0c16b5e8adf0e0acaf7f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Apr 2023 20:44:56 +0200 Subject: [PATCH 109/194] refactor(sqlserver): support only sync mode at connector level --- .../src/emqx_ee_connector_sqlserver.erl | 52 ++----------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 6cbd9de4e..97a46152d 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -34,8 +34,6 @@ on_stop/2, on_query/3, on_batch_query/3, - on_query_async/4, - on_batch_query_async/4, on_get_status/2 ]). @@ -43,7 +41,7 @@ -export([connect/1]). %% Internal exports used to execute code with ecpool worker --export([do_get_status/1, worker_do_insert/3, do_async_reply/2]). +-export([do_get_status/1, worker_do_insert/3]). -import(emqx_plugin_libs_rule, [str/1]). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -51,7 +49,6 @@ -define(ACTION_SEND_MESSAGE, send_message). -define(SYNC_QUERY_MODE, handover). --define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). -define(SQLSERVER_HOST_OPTIONS, #{ default_port => 1433 @@ -169,7 +166,7 @@ server() -> %% Callbacks defined in emqx_resource %%==================================================================== -callback_mode() -> async_if_possible. +callback_mode() -> always_sync. is_buffer_supported() -> false. @@ -253,28 +250,6 @@ on_query(InstanceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> ), do_query(InstanceId, Query, ?SYNC_QUERY_MODE, State). --spec on_query_async( - manager_id(), - {?ACTION_SEND_MESSAGE, map()}, - {ReplyFun :: function(), Args :: list()}, - state() -) -> - {ok, any()} - | {error, term()}. -on_query_async( - InstanceId, - {?ACTION_SEND_MESSAGE, _Msg} = Query, - ReplyFunAndArgs, - %% #{poolname := PoolName, sql_templates := Templates} = State - State -) -> - ?TRACE( - "SINGLE_QUERY_ASYNC", - "bridge_sqlserver_received", - #{requests => Query, connector => InstanceId, state => State} - ), - do_query(InstanceId, Query, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). - -spec on_batch_query( manager_id(), [{?ACTION_SEND_MESSAGE, map()}], @@ -292,20 +267,6 @@ on_batch_query(InstanceId, BatchRequests, State) -> ), do_query(InstanceId, BatchRequests, ?SYNC_QUERY_MODE, State). --spec on_batch_query_async( - manager_id(), - [{?ACTION_SEND_MESSAGE, map()}], - {ReplyFun :: function(), Args :: list()}, - state() -) -> {ok, any()}. -on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> - ?TRACE( - "BATCH_QUERY_ASYNC", - "bridge_sqlserver_received", - #{requests => Requests, connector => InstanceId, state => State} - ), - do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). - on_get_status(_InstanceId, #{poolname := Pool} = _State) -> Health = emqx_plugin_libs_pool:health_check_ecpool_workers( Pool, {?MODULE, do_get_status, []} @@ -365,13 +326,11 @@ conn_str([{password, Password} | Opts], Acc) -> conn_str([{_, _} | Opts], Acc) -> conn_str(Opts, Acc). -%% Sync & Async query with singe & batch sql statement +%% Query with singe & batch sql statement -spec do_query( manager_id(), Query :: {?ACTION_SEND_MESSAGE, map()} | [{?ACTION_SEND_MESSAGE, map()}], - ApplyMode :: - handover - | {handover_async, {?MODULE, do_async_reply, [{ReplyFun :: function(), Args :: list()}]}}, + ApplyMode :: handover, state() ) -> {ok, list()} @@ -531,6 +490,3 @@ apply_template(Query, Templates) -> %% TODO: more detail infomatoin ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}), {error, failed_to_apply_sql_template}. - -do_async_reply(Result, {ReplyFun, Args}) -> - erlang:apply(ReplyFun, Args ++ [Result]). From 51cd83e70fc12b8092b826c82eb9b4d43d53b961 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 18 Jan 2023 08:12:43 +0100 Subject: [PATCH 110/194] refactor: delete dead code --- apps/emqx_machine/src/emqx_machine.erl | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 6872b150c..aa8f03ae5 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -43,7 +43,7 @@ start() -> start_sysmon(), configure_shard_transports(), ekka:start(), - ok = print_otp_version_warning(). + ok. graceful_shutdown() -> emqx_machine_terminator:graceful_wait(). @@ -61,17 +61,6 @@ set_backtrace_depth() -> is_ready() -> emqx_machine_terminator:is_running(). --if(?OTP_RELEASE > 22). -print_otp_version_warning() -> ok. --else. -print_otp_version_warning() -> - ?ULOG( - "WARNING: Running on Erlang/OTP version ~p. Recommended: 23~n", - [?OTP_RELEASE] - ). -% OTP_RELEASE > 22 --endif. - start_sysmon() -> _ = application:load(system_monitor), application:set_env(system_monitor, node_status_fun, {?MODULE, node_status}), From d89975c6eedaa6bbc61c4799faa2356bd98f70da Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 18 Jan 2023 08:20:56 +0100 Subject: [PATCH 111/194] test: add a test case for emqx_machine:node_status --- apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/test/emqx_machine_SUITE.erl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 6bd36aab5..a44d2b36e 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 691cda677..02d03d983 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -103,3 +103,13 @@ t_custom_shard_transports(_Config) -> emqx_machine:start(), ?assertEqual(distr, mria_config:shard_transport(Shard)), ok. + +t_node_status(_Config) -> + JSON = emqx_machine:node_status(), + ?assertMatch( + #{ + <<"backend">> := _, + <<"role">> := <<"core">> + }, + jsx:decode(JSON) + ). From 0c284ce5fed6e62dccf03289fc4458c5e86fb95f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 26 Apr 2023 20:50:06 +0200 Subject: [PATCH 112/194] chore: bump app versions --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 03cd36522..e6c3d9cd9 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.18"}, + {vsn, "0.1.19"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 8c7e424e0..bd022f226 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.19"}, + {vsn, "5.0.20"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index ec282b60b..34f3dd1fe 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, From 7e24b35bb3bc5b5735af67a7335d8a507f21fefe Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 26 Apr 2023 17:09:39 -0300 Subject: [PATCH 113/194] chore: bump release version to `e5.0.3-alpha.4` --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 6d91b2528..9e7e53b32 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3-alpha.3"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.4"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 262f0e8dd9cd4a8a08612cee293ad3e11c628639 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 27 Apr 2023 10:28:05 +0800 Subject: [PATCH 114/194] chore: update README --- apps/emqx_bridge_matrix/README.md | 3 +-- apps/emqx_bridge_mysql/README.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge_matrix/README.md b/apps/emqx_bridge_matrix/README.md index 339eb0605..0d9c4fc4a 100644 --- a/apps/emqx_bridge_matrix/README.md +++ b/apps/emqx_bridge_matrix/README.md @@ -1,7 +1,6 @@ # EMQX MatrixDB Bridge -[MatrixDB](http://matrixdb.univ-lyon1.fr/) is a biological database focused on -molecular interactions between extracellular proteins and polysaccharides. +[YMatrix](https://www.ymatrix.cn/) is a hyper-converged database product developed by YMatrix based on the PostgreSQL / Greenplum classic open source database. In addition to being able to handle time series scenarios with ease, it also supports classic scenarios such as online transaction processing (OLTP) and online analytical processing (OLAP). The application is used to connect EMQX and MatrixDB. User can create a rule and easily ingest IoT data into MatrixDB by leveraging diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md index 73f6987b6..d7c9b5647 100644 --- a/apps/emqx_bridge_mysql/README.md +++ b/apps/emqx_bridge_mysql/README.md @@ -1,6 +1,6 @@ # EMQX MySQL Bridge -[MySQL](https://github.com/MySQL/MySQL) is a popular open-source relational database +[MySQL](https://github.com/mysql/mysql-server) is a popular open-source relational database management system. The application is used to connect EMQX and MySQL. From ce2f2217eecfd76884ddda9b1d0ebcefbe19b807 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 27 Apr 2023 10:36:50 +0800 Subject: [PATCH 115/194] chore: bump versions --- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index 03cd36522..e6c3d9cd9 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.18"}, + {vsn, "0.1.19"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 8c7e424e0..bd022f226 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.19"}, + {vsn, "5.0.20"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index ec282b60b..34f3dd1fe 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, From e789e57b653e78952d4acbed858f7d9f56228213 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 27 Apr 2023 14:41:23 +0800 Subject: [PATCH 116/194] chore: change node.data_dir from hidden to low --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 8e5e6937f..39dce0b71 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -471,7 +471,7 @@ fields("node") -> %% for now, it's tricky to use a different data_dir %% otherwise data paths in cluster config may differ %% TODO: change configurable data file paths to relative - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(node_data_dir) } )}, From 99448151e9e1928c4d16303ca90dd8ff6b1e7dff Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 26 Apr 2023 09:38:15 -0300 Subject: [PATCH 117/194] test(crl): ensure ssl_manager is ready to avoid flakiness Example failure: https://github.com/emqx/emqx/actions/runs/4806430125/jobs/8555021522?pr=10524#step:8:49138 --- apps/emqx/test/emqx_crl_cache_SUITE.erl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index dd3eb29e7..1b8abb9c3 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -35,6 +35,7 @@ all() -> init_per_suite(Config) -> application:load(emqx), + {ok, _} = application:ensure_all_started(ssl), emqx_config:save_schema_mod_and_names(emqx_schema), emqx_common_test_helpers:boot_modules(all), Config. @@ -328,7 +329,15 @@ drain_msgs() -> clear_crl_cache() -> %% reset the CRL cache + Ref = monitor(process, whereis(ssl_manager)), exit(whereis(ssl_manager), kill), + receive + {'DOWN', Ref, process, _, _} -> + ok + after 1_000 -> + ct:fail("ssl_manager didn't die") + end, + ensure_ssl_manager_alive(), ok. force_cacertfile(Cacertfile) -> @@ -382,7 +391,6 @@ setup_crl_options(Config, #{is_cached := IsCached} = Opts) -> false -> %% ensure cache is empty clear_crl_cache(), - ct:sleep(200), ok end, drain_msgs(), @@ -459,6 +467,13 @@ of_kinds(Trace0, Kinds0) -> Trace0 ). +ensure_ssl_manager_alive() -> + ?retry( + _Sleep0 = 200, + _Attempts0 = 50, + true = is_pid(whereis(ssl_manager)) + ). + %%-------------------------------------------------------------------- %% Test cases %%-------------------------------------------------------------------- From 66155a8636e59f6ba5edb3af2985a70ef0abf39e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 26 Apr 2023 17:55:33 +0800 Subject: [PATCH 118/194] chore: add ms msodbcsql docker file base on emqx-enterprise --- deploy/docker/Dockerfile.msodbc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 deploy/docker/Dockerfile.msodbc diff --git a/deploy/docker/Dockerfile.msodbc b/deploy/docker/Dockerfile.msodbc new file mode 100644 index 000000000..d7b3457ac --- /dev/null +++ b/deploy/docker/Dockerfile.msodbc @@ -0,0 +1,25 @@ +## This Dockerfile should not run in GitHub Action or any other automated process. +## It should be manually executed by the needs of the user. +## +## Before manaually execute: +## Please confirm the EMQX-Enterprise version you are using and modify the base layer image tag +## ```bash +## $ docker build -f=Dockerfile.msodbc -t emqx-enterprise-with-msodbc:5.0.3-alpha.2 . +## ``` + +# FROM emqx/emqx-enterprise:latest +FROM emqx/emqx-enterprise:5.0.3-alpha.2 + +USER root + +RUN apt-get update \ + && apt-get install -y gnupg2 curl apt-utils \ + && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + && curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-mkc crelease.list \ + && apt-get update \ + && ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev \ + && sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +USER emqx From 0f37f38fda991e7c3943fd2fdda72bf5d9404698 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 09:51:40 -0300 Subject: [PATCH 119/194] ci: set `IS_CI=yes` when running tests --- .github/workflows/run_test_cases.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index f7b775f08..0ebc67e13 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -14,6 +14,9 @@ on: - e* pull_request: +env: + IS_CI: "yes" + jobs: build-matrix: runs-on: ubuntu-22.04 From 7853a4c36ee7a40697faf121250ca4873e720d0b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 11:58:28 -0300 Subject: [PATCH 120/194] chore: bump app vsns --- apps/emqx_resource/src/emqx_resource.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 2553e6dd8..3e264cb3e 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ From 521b54904985448021f99847783a84c1468a7e32 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 11:58:40 -0300 Subject: [PATCH 121/194] test(peer): define cookie when using `ct_slave` module --- apps/emqx/test/emqx_common_test_helpers.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ac03f4660..71e1bee84 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -271,6 +271,7 @@ app_schema(App) -> mustache_vars(App, Opts) -> ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}), Defaults = #{ + node_cookie => atom_to_list(erlang:get_cookie()), platform_data_dir => app_path(App, "data"), platform_etc_dir => app_path(App, "etc"), platform_log_dir => app_path(App, "log") @@ -667,6 +668,7 @@ start_slave(Name, Opts) when is_map(Opts) -> SlaveMod = maps:get(peer_mod, Opts, ct_slave), Node = node_name(Name), put_peer_mod(Node, SlaveMod), + Cookie = atom_to_list(erlang:get_cookie()), DoStart = fun() -> case SlaveMod of @@ -678,7 +680,11 @@ start_slave(Name, Opts) when is_map(Opts) -> {monitor_master, true}, {init_timeout, 20_000}, {startup_timeout, 20_000}, - {erl_flags, erl_flags()} + {erl_flags, erl_flags()}, + {env, [ + {"HOCON_ENV_OVERRIDE_PREFIX", "EMQX_"}, + {"EMQX_NODE__COOKIE", Cookie} + ]} ] ); slave -> From dd90b2f49827a6f479ff7cb526d1daa9a5e03862 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Mon, 24 Apr 2023 12:21:02 -0300 Subject: [PATCH 122/194] feat(oracle): Oracle Database integration --- .../docker-compose-oracle.yaml | 11 + .ci/docker-compose-file/toxiproxy.json | 6 + apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 3 +- apps/emqx_bridge_oracle/BSL.txt | 94 +++ apps/emqx_bridge_oracle/README.md | 28 + apps/emqx_bridge_oracle/docker-ct | 2 + .../etc/emqx_bridge_oracle.conf | 0 apps/emqx_bridge_oracle/rebar.config | 13 + .../src/emqx_bridge_oracle.app.src | 14 + .../src/emqx_bridge_oracle.erl | 109 ++++ .../test/emqx_bridge_oracle_SUITE.erl | 594 ++++++++++++++++++ apps/emqx_oracle/BSL.txt | 94 +++ apps/emqx_oracle/README.md | 14 + apps/emqx_oracle/rebar.config | 7 + apps/emqx_oracle/src/emqx_oracle.app.src | 14 + apps/emqx_oracle/src/emqx_oracle.erl | 434 +++++++++++++ apps/emqx_oracle/src/emqx_oracle_schema.erl | 33 + .../emqx_plugin_libs/src/emqx_placeholder.erl | 11 +- .../src/emqx_plugin_libs.app.src | 2 +- .../src/emqx_plugin_libs_rule.erl | 3 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- changes/ee/feat-10498.en.md | 1 + .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 17 +- mix.exs | 6 +- rebar.config.erl | 4 + rel/i18n/emqx_bridge_oracle.hocon | 52 ++ rel/i18n/emqx_oracle.hocon | 15 + rel/i18n/zh/emqx_bridge_oracle.hocon | 51 ++ rel/i18n/zh/emqx_oracle.hocon | 15 + scripts/ct/run.sh | 3 + 32 files changed, 1642 insertions(+), 14 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-oracle.yaml create mode 100644 apps/emqx_bridge_oracle/BSL.txt create mode 100644 apps/emqx_bridge_oracle/README.md create mode 100644 apps/emqx_bridge_oracle/docker-ct create mode 100644 apps/emqx_bridge_oracle/etc/emqx_bridge_oracle.conf create mode 100644 apps/emqx_bridge_oracle/rebar.config create mode 100644 apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src create mode 100644 apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl create mode 100644 apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl create mode 100644 apps/emqx_oracle/BSL.txt create mode 100644 apps/emqx_oracle/README.md create mode 100644 apps/emqx_oracle/rebar.config create mode 100644 apps/emqx_oracle/src/emqx_oracle.app.src create mode 100644 apps/emqx_oracle/src/emqx_oracle.erl create mode 100644 apps/emqx_oracle/src/emqx_oracle_schema.erl create mode 100644 changes/ee/feat-10498.en.md create mode 100644 rel/i18n/emqx_bridge_oracle.hocon create mode 100644 rel/i18n/emqx_oracle.hocon create mode 100644 rel/i18n/zh/emqx_bridge_oracle.hocon create mode 100644 rel/i18n/zh/emqx_oracle.hocon diff --git a/.ci/docker-compose-file/docker-compose-oracle.yaml b/.ci/docker-compose-file/docker-compose-oracle.yaml new file mode 100644 index 000000000..ea8965846 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-oracle.yaml @@ -0,0 +1,11 @@ +version: '3.9' + +services: + oracle_server: + container_name: oracle + image: oracleinanutshell/oracle-xe-11g:1.0.0 + restart: always + environment: + ORACLE_DISABLE_ASYNCH_IO: true + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 9cefcb808..e4fbfa62a 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -119,5 +119,11 @@ "listen": "0.0.0.0:6653", "upstream": "pulsar:6653", "enabled": true + }, + { + "name": "oracle", + "listen": "0.0.0.0:1521", + "upstream": "oracle:1521", + "enabled": true } ] diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index d6c140fef..e408250be 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index fd4e16263..a37b6db3c 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -71,7 +71,8 @@ T == rocketmq; T == cassandra; T == sqlserver; - T == pulsar_producer + T == pulsar_producer; + T == oracle ). load() -> diff --git a/apps/emqx_bridge_oracle/BSL.txt b/apps/emqx_bridge_oracle/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_oracle/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_oracle/README.md b/apps/emqx_bridge_oracle/README.md new file mode 100644 index 000000000..d2974b722 --- /dev/null +++ b/apps/emqx_bridge_oracle/README.md @@ -0,0 +1,28 @@ +# EMQX Oracle Database Bridge + +This application houses the Oracle Database bridge for EMQX Enterprise Edition. +It implements the data bridge APIs for interacting with an Oracle Database Bridge. + + +# Documentation + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +## License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_oracle/docker-ct b/apps/emqx_bridge_oracle/docker-ct new file mode 100644 index 000000000..c24dc4bc9 --- /dev/null +++ b/apps/emqx_bridge_oracle/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +oracle diff --git a/apps/emqx_bridge_oracle/etc/emqx_bridge_oracle.conf b/apps/emqx_bridge_oracle/etc/emqx_bridge_oracle.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_oracle/rebar.config b/apps/emqx_bridge_oracle/rebar.config new file mode 100644 index 000000000..c238546c4 --- /dev/null +++ b/apps/emqx_bridge_oracle/rebar.config @@ -0,0 +1,13 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ {emqx_oracle, {path, "../../apps/emqx_oracle"}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_bridge_oracle]} +]}. diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src new file mode 100644 index 000000000..4f81c2110 --- /dev/null +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -0,0 +1,14 @@ +{application, emqx_bridge_oracle, [ + {description, "EMQX Enterprise Oracle Database Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + emqx_oracle + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl new file mode 100644 index 000000000..8a87f02ba --- /dev/null +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_oracle). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into t_mqtt_msg(msgid, topic, qos, payload)" + "values (${id}, ${topic}, ${qos}, ${payload})" +>>). + +conn_bridge_examples(Method) -> + [ + #{ + <<"oracle">> => #{ + summary => <<"Oracle Database Bridge">>, + value => values(Method) + } + } + ]. + +values(_Method) -> + #{ + enable => true, + type => oracle, + name => <<"foo">>, + server => <<"127.0.0.1:1521">>, + pool_size => 8, + database => <<"ORCL">>, + sid => <<"ORCL">>, + username => <<"root">>, + password => <<"******">>, + sql => ?DEFAULT_SQL, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES + } + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions + +namespace() -> "bridge_oracle". + +roots() -> []. + +fields("config") -> + [ + {enable, + hoconsc:mk( + boolean(), + #{desc => ?DESC("config_enable"), default => true} + )}, + {sql, + hoconsc:mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {local_topic, + hoconsc:mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )} + ] ++ emqx_resource_schema:fields("resource_opts") ++ + (emqx_oracle_schema:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()); +fields("post") -> + fields("post", oracle); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field(Type) -> + {type, hoconsc:mk(hoconsc:enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, hoconsc:mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl new file mode 100644 index 000000000..de77b26de --- /dev/null +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -0,0 +1,594 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_oracle_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(BRIDGE_TYPE_BIN, <<"oracle">>). +-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_oracle, emqx_bridge_oracle]). +-define(DATABASE, "XE"). +-define(RULE_TOPIC, "mqtt/rule"). +% -define(RULE_TOPIC_BIN, <>). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, plain} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + [ + {plain, AllTCs} + ]. + +only_once_tests() -> + [t_create_via_http]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), + _ = application:stop(emqx_connector), + ok. + +init_per_group(plain = Type, Config) -> + OracleHost = os:getenv("ORACLE_PLAIN_HOST", "toxiproxy.emqx.net"), + OraclePort = list_to_integer(os:getenv("ORACLE_PLAIN_PORT", "1521")), + ProxyName = "oracle", + case emqx_common_test_helpers:is_tcp_server_available(OracleHost, OraclePort) of + true -> + Config1 = common_init_per_group(), + [ + {proxy_name, ProxyName}, + {oracle_host, OracleHost}, + {oracle_port, OraclePort}, + {connection_type, Type} + | Config1 ++ Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_oracle); + _ -> + {skip, no_oracle} + end + end; +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when + Group =:= plain +-> + common_end_per_group(Config), + ok; +end_per_group(_Group, _Config) -> + ok. + +common_init_per_group() -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + application:load(emqx_bridge), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps(?APPS), + {ok, _} = application:ensure_all_started(emqx_connector), + emqx_mgmt_api_test_util:init_suite(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + MQTTTopic = <<"mqtt/topic/", UniqueNum/binary>>, + [ + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {mqtt_topic, MQTTTopic} + ]. + +common_end_per_group(Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + ok. + +init_per_testcase(TestCase, Config) -> + common_init_per_testcase(TestCase, Config). + +end_per_testcase(_Testcase, Config) -> + common_end_per_testcase(_Testcase, Config). + +common_init_per_testcase(TestCase, Config0) -> + ct:timetrap(timer:seconds(60)), + delete_all_bridges(), + UniqueNum = integer_to_binary(erlang:unique_integer()), + OracleTopic = + << + (atom_to_binary(TestCase))/binary, + UniqueNum/binary + >>, + ConnectionType = ?config(connection_type, Config0), + Config = [{oracle_topic, OracleTopic} | Config0], + {Name, ConfigString, OracleConfig} = oracle_config( + TestCase, ConnectionType, Config + ), + ok = snabbkaffe:start_trace(), + [ + {oracle_name, Name}, + {oracle_config_string, ConfigString}, + {oracle_config, OracleConfig} + | Config + ]. + +common_end_per_testcase(_Testcase, Config) -> + case proplists:get_bool(skip_does_not_apply, Config) of + true -> + ok; + false -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), + ok = snabbkaffe:stop(), + ok + end. + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ +sql_insert_template_for_bridge() -> + "INSERT INTO mqtt_test(topic, msgid, payload, retain) VALUES (${topic}, ${id}, ${payload}, ${retain})". + +sql_create_table() -> + "CREATE TABLE mqtt_test (topic VARCHAR2(255), msgid VARCHAR2(64), payload NCLOB, retain NUMBER(1))". + +sql_drop_table() -> + "DROP TABLE mqtt_test". + +reset_table(Config) -> + ResourceId = resource_id(Config), + _ = emqx_resource:simple_sync_query(ResourceId, {sql, sql_drop_table()}), + {ok, [{proc_result, 0, _}]} = emqx_resource:simple_sync_query( + ResourceId, {sql, sql_create_table()} + ), + ok. + +drop_table(Config) -> + ResourceId = resource_id(Config), + emqx_resource:simple_sync_query(ResourceId, {query, sql_drop_table()}), + ok. + +oracle_config(TestCase, _ConnectionType, Config) -> + UniqueNum = integer_to_binary(erlang:unique_integer()), + OracleHost = ?config(oracle_host, Config), + OraclePort = ?config(oracle_port, Config), + Name = << + (atom_to_binary(TestCase))/binary, UniqueNum/binary + >>, + ServerURL = iolist_to_binary([ + OracleHost, + ":", + integer_to_binary(OraclePort) + ]), + ConfigString = + io_lib:format( + "bridges.oracle.~s {\n" + " enable = true\n" + " database = \"~s\"\n" + " sid = \"~s\"\n" + " server = \"~s\"\n" + " username = \"system\"\n" + " password = \"oracle\"\n" + " pool_size = 1\n" + " sql = \"~s\"\n" + " resource_opts = {\n" + " auto_restart_interval = 5000\n" + " request_timeout = 30000\n" + " query_mode = \"async\"\n" + " enable_batch = true\n" + " batch_size = 3\n" + " batch_time = \"3s\"\n" + " worker_pool_size = 1\n" + " }\n" + "}\n", + [ + Name, + ?DATABASE, + ?DATABASE, + ServerURL, + sql_insert_template_for_bridge() + ] + ), + {Name, ConfigString, parse_and_check(ConfigString, Name)}. + +parse_and_check(ConfigString, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + TypeBin = ?BRIDGE_TYPE_BIN, + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf, + Config. + +resource_id(Config) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(oracle_name, Config), + emqx_bridge_resource:resource_id(Type, Name). + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + Type = ?BRIDGE_TYPE_BIN, + Name = ?config(oracle_name, Config), + OracleConfig0 = ?config(oracle_config, Config), + OracleConfig = emqx_utils_maps:deep_merge(OracleConfig0, Overrides), + emqx_bridge:create(Type, Name, OracleConfig). + +create_bridge_api(Config) -> + create_bridge_api(Config, _Overrides = #{}). + +create_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(oracle_name, Config), + OracleConfig0 = ?config(oracle_config, Config), + OracleConfig = emqx_utils_maps:deep_merge(OracleConfig0, Overrides), + Params = OracleConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("creating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {Status, Headers, Body0}} -> + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; + Error -> + Error + end, + ct:pal("bridge create result: ~p", [Res]), + Res. + +update_bridge_api(Config) -> + update_bridge_api(Config, _Overrides = #{}). + +update_bridge_api(Config, Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(oracle_name, Config), + OracleConfig0 = ?config(oracle_config, Config), + OracleConfig = emqx_utils_maps:deep_merge(OracleConfig0, Overrides), + BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), + Params = OracleConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("updating bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; + Error -> Error + end, + ct:pal("bridge update result: ~p", [Res]), + Res. + +probe_bridge_api(Config) -> + probe_bridge_api(Config, _Overrides = #{}). + +probe_bridge_api(Config, _Overrides) -> + TypeBin = ?BRIDGE_TYPE_BIN, + Name = ?config(oracle_name, Config), + OracleConfig = ?config(oracle_config, Config), + Params = OracleConfig#{<<"type">> => TypeBin, <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Opts = #{return_all => true}, + ct:pal("probing bridge (via http): ~p", [Params]), + Res = + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, 204, _}, _Headers, _Body0} = Res0} -> {ok, Res0}; + Error -> Error + end, + ct:pal("bridge probe result: ~p", [Res]), + Res. + +create_rule_and_action_http(Config) -> + OracleName = ?config(oracle_name, Config), + BridgeId = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE_BIN, OracleName), + Params = #{ + enable => true, + sql => <<"SELECT * FROM \"", ?RULE_TOPIC, "\"">>, + actions => [BridgeId] + }, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ct:pal("rule action params: ~p", [Params]), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +% Under normal operations, the bridge will be called async via +% `simple_async_query'. +t_sync_query(Config) -> + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + reset_table(Config), + MsgId = erlang:unique_integer(), + Params = #{ + topic => ?config(mqtt_topic, Config), + id => MsgId, + payload => ?config(oracle_name, Config), + retain => true + }, + Message = {send_message, Params}, + ?assertEqual( + {ok, [{affected_rows, 1}]}, emqx_resource:simple_sync_query(ResourceId, Message) + ), + ok + end, + [] + ), + ok. + +t_async_query(Config) -> + Overrides = #{ + <<"resource_opts">> => #{ + <<"enable_batch">> => <<"false">>, + <<"batch_size">> => 1 + } + }, + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config, Overrides)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + reset_table(Config), + MsgId = erlang:unique_integer(), + Params = #{ + topic => ?config(mqtt_topic, Config), + id => MsgId, + payload => ?config(oracle_name, Config), + retain => false + }, + Message = {send_message, Params}, + ?assertMatch( + { + ok, + {ok, #{result := {ok, [{affected_rows, 1}]}}} + }, + ?wait_async_action( + emqx_resource:query(ResourceId, Message), + #{?snk_kind := oracle_query}, + 5_000 + ) + ), + ok + end, + [] + ), + ok. + +t_batch_sync_query(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 30, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + reset_table(Config), + MsgId = erlang:unique_integer(), + Params = #{ + topic => ?config(mqtt_topic, Config), + id => MsgId, + payload => ?config(oracle_name, Config), + retain => false + }, + % Send 3 async messages while resource is down. When it comes back, these messages + % will be delivered in sync way. If we try to send sync messages directly, it will + % be sent async as callback_mode is set to async_if_possible. + Message = {send_message, Params}, + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ct:sleep(1000), + emqx_resource:query(ResourceId, Message), + emqx_resource:query(ResourceId, Message), + emqx_resource:query(ResourceId, Message) + end), + ?retry( + _Sleep = 1_000, + _Attempts = 30, + ?assertMatch( + {ok, [{result_set, _, _, [[{3}]]}]}, + emqx_resource:simple_sync_query( + ResourceId, {query, "SELECT COUNT(*) FROM mqtt_test"} + ) + ) + ), + ok + end, + [] + ), + ok. + +t_batch_async_query(Config) -> + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + reset_table(Config), + MsgId = erlang:unique_integer(), + Params = #{ + topic => ?config(mqtt_topic, Config), + id => MsgId, + payload => ?config(oracle_name, Config), + retain => false + }, + Message = {send_message, Params}, + ?assertMatch( + { + ok, + {ok, #{result := {ok, [{affected_rows, 1}]}}} + }, + ?wait_async_action( + emqx_resource:query(ResourceId, Message), + #{?snk_kind := oracle_batch_query}, + 5_000 + ) + ), + ok + end, + [] + ), + ok. + +t_create_via_http(Config) -> + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge_api(Config)), + + %% lightweight matrix testing some configs + ?assertMatch( + {ok, _}, + update_bridge_api( + Config, + #{ + <<"resource_opts">> => + #{<<"batch_size">> => 4} + } + ) + ), + ?assertMatch( + {ok, _}, + update_bridge_api( + Config, + #{ + <<"resource_opts">> => + #{<<"batch_time">> => <<"4s">>} + } + ) + ), + ok + end, + [] + ), + ok. + +t_start_stop(Config) -> + OracleName = ?config(oracle_name, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + ?assertMatch({ok, _}, create_bridge(Config)), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + + %% Check that the bridge probe API doesn't leak atoms. + ProbeRes0 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes0), + AtomsBefore = erlang:system_info(atom_count), + %% Probe again; shouldn't have created more atoms. + ProbeRes1 = probe_bridge_api( + Config, + #{<<"resource_opts">> => #{<<"health_check_interval">> => <<"1s">>}} + ), + + ?assertMatch({ok, {{_, 204, _}, _Headers, _Body}}, ProbeRes1), + AtomsAfter = erlang:system_info(atom_count), + ?assertEqual(AtomsBefore, AtomsAfter), + + %% Now stop the bridge. + ?assertMatch( + {{ok, _}, {ok, _}}, + ?wait_async_action( + emqx_bridge:disable_enable(disable, ?BRIDGE_TYPE_BIN, OracleName), + #{?snk_kind := oracle_bridge_stopped}, + 5_000 + ) + ), + + ok + end, + fun(Trace) -> + %% one for each probe, one for real + ?assertMatch([_, _, _], ?of_kind(oracle_bridge_stopped, Trace)), + ok + end + ), + ok. + +t_on_get_status(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ResourceId = resource_id(Config), + ?assertMatch({ok, _}, create_bridge(Config)), + %% Since the connection process is async, we give it some time to + %% stabilize and avoid flakiness. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ct:sleep(500), + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)) + end), + %% Check that it recovers itself. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + ok. diff --git a/apps/emqx_oracle/BSL.txt b/apps/emqx_oracle/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_oracle/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_oracle/README.md b/apps/emqx_oracle/README.md new file mode 100644 index 000000000..873d52259 --- /dev/null +++ b/apps/emqx_oracle/README.md @@ -0,0 +1,14 @@ +# Oracle Database Connector + +This application houses the Oracle Database connector for EMQX Enterprise Edition. +It provides the APIs to connect to Oracle Database. + +So far it is only used to insert messages as data bridge. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_oracle/rebar.config b/apps/emqx_oracle/rebar.config new file mode 100644 index 000000000..14461ba34 --- /dev/null +++ b/apps/emqx_oracle/rebar.config @@ -0,0 +1,7 @@ +%% -*- mode: erlang; -*- + +{erl_opts, [debug_info]}. +{deps, [ {jamdb_oracle, {git, "https://github.com/emqx/jamdb_oracle", {tag, "0.4.9.4"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src new file mode 100644 index 000000000..fa48e8479 --- /dev/null +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -0,0 +1,14 @@ +{application, emqx_oracle, [ + {description, "EMQX Enterprise Oracle Database Connector"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + jamdb_oracle + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl new file mode 100644 index 000000000..c39a6a6d7 --- /dev/null +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -0,0 +1,434 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_oracle). + +-behaviour(emqx_resource). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(ORACLE_DEFAULT_PORT, 1521). + +%%==================================================================== +%% Exports +%%==================================================================== + +%% callbacks for behaviour emqx_resource +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_query_async/4, + on_batch_query_async/4, + on_get_status/2 +]). + +%% callbacks for ecpool +-export([connect/1, prepare_sql_to_conn/2]). + +%% Internal exports used to execute code with ecpool worker +-export([ + query/3, + execute_batch/3, + do_async_reply/2, + do_get_status/1 +]). + +-export([ + oracle_host_options/0 +]). + +-define(ACTION_SEND_MESSAGE, send_message). + +-define(SYNC_QUERY_MODE, no_handover). +-define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). + +-define(ORACLE_HOST_OPTIONS, #{ + default_port => ?ORACLE_DEFAULT_PORT +}). + +-define(MAX_CURSORS, 10). +-define(DEFAULT_POOL_SIZE, 8). +-define(OPT_TIMEOUT, 30000). + +-type prepares() :: #{atom() => binary()}. +-type params_tokens() :: #{atom() => list()}. + +-type state() :: + #{ + pool_name := binary(), + prepare_sql := prepares(), + params_tokens := params_tokens(), + batch_params_tokens := params_tokens() + }. + +callback_mode() -> async_if_possible. + +is_buffer_supported() -> false. + +-spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +on_start( + InstId, + #{ + server := Server, + database := DB, + sid := Sid, + username := User + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_oracle_connector", + connector => InstId, + config => emqx_utils:redact(Config) + }), + ?tp(oracle_bridge_started, #{instance_id => InstId, config => Config}), + {ok, _} = application:ensure_all_started(ecpool), + {ok, _} = application:ensure_all_started(jamdb_oracle), + jamdb_oracle_conn:set_max_cursors_number(?MAX_CURSORS), + + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, oracle_host_options()), + ServiceName = maps:get(<<"service_name">>, Config, Sid), + Options = [ + {host, Host}, + {port, Port}, + {user, emqx_plugin_libs_rule:str(User)}, + {password, emqx_secret:wrap(maps:get(password, Config, ""))}, + {sid, emqx_plugin_libs_rule:str(Sid)}, + {service_name, emqx_plugin_libs_rule:str(ServiceName)}, + {database, DB}, + {pool_size, maps:get(<<"pool_size">>, Config, ?DEFAULT_POOL_SIZE)}, + {timeout, ?OPT_TIMEOUT}, + {app_name, "EMQX Data To Oracle Database Action"} + ], + PoolName = InstId, + Prepares = parse_prepare_sql(Config), + InitState = #{pool_name => PoolName, prepare_statement => #{}}, + State = maps:merge(InitState, Prepares), + case emqx_resource_pool:start(InstId, ?MODULE, Options) of + ok -> + {ok, init_prepare(State)}; + {error, Reason} -> + ?tp( + oracle_connector_start_failed, + #{error => Reason} + ), + {error, Reason} + end. + +on_stop(InstId, #{pool_name := PoolName}) -> + ?SLOG(info, #{ + msg => "stopping_oracle_connector", + connector => InstId + }), + ?tp(oracle_bridge_stopped, #{instance_id => InstId}), + emqx_resource_pool:stop(PoolName). + +on_query(InstId, {TypeOrKey, NameOrSQL}, #{pool_name := _PoolName} = State) -> + on_query(InstId, {TypeOrKey, NameOrSQL, []}, State); +on_query( + InstId, + {TypeOrKey, NameOrSQL, Params}, + #{pool_name := PoolName} = State +) -> + ?SLOG(debug, #{ + msg => "oracle database connector received sql query", + connector => InstId, + type => TypeOrKey, + sql => NameOrSQL, + state => State + }), + Type = query, + {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), + Res = on_sql_query(InstId, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), + handle_result(Res). + +on_query_async(InstId, {TypeOrKey, NameOrSQL}, Reply, State) -> + on_query_async(InstId, {TypeOrKey, NameOrSQL, []}, Reply, State); +on_query_async( + InstId, {TypeOrKey, NameOrSQL, Params} = Query, Reply, #{pool_name := PoolName} = State +) -> + ?SLOG(debug, #{ + msg => "oracle database connector received async sql query", + connector => InstId, + query => Query, + reply => Reply, + state => State + }), + ApplyMode = ?ASYNC_QUERY_MODE(Reply), + Type = query, + {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), + Res = on_sql_query(InstId, PoolName, Type, ApplyMode, NameOrSQL2, Data), + handle_result(Res). + +on_batch_query( + InstId, + BatchReq, + #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State +) -> + case BatchReq of + [{Key, _} = Request | _] -> + BinKey = to_bin(Key), + case maps:get(BinKey, Tokens, undefined) of + undefined -> + Log = #{ + connector => InstId, + first_request => Request, + state => State, + msg => "batch prepare not implemented" + }, + ?SLOG(error, Log), + {error, {unrecoverable_error, batch_prepare_not_implemented}}; + TokenList -> + {_, Datas} = lists:unzip(BatchReq), + Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas], + St = maps:get(BinKey, Sts), + case + on_sql_query(InstId, PoolName, execute_batch, ?SYNC_QUERY_MODE, St, Datas2) + of + {ok, Results} -> + handle_batch_result(Results, 0); + Result -> + Result + end + end; + _ -> + Log = #{ + connector => InstId, + request => BatchReq, + state => State, + msg => "invalid request" + }, + ?SLOG(error, Log), + {error, {unrecoverable_error, invalid_request}} + end. + +on_batch_query_async( + InstId, + BatchReq, + Reply, + #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State +) -> + case BatchReq of + [{Key, _} = Request | _] -> + BinKey = to_bin(Key), + case maps:get(BinKey, Tokens, undefined) of + undefined -> + Log = #{ + connector => InstId, + first_request => Request, + state => State, + msg => "batch prepare not implemented" + }, + ?SLOG(error, Log), + {error, {unrecoverable_error, batch_prepare_not_implemented}}; + TokenList -> + {_, Datas} = lists:unzip(BatchReq), + Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas], + St = maps:get(BinKey, Sts), + case + on_sql_query( + InstId, PoolName, execute_batch, ?ASYNC_QUERY_MODE(Reply), St, Datas2 + ) + of + {ok, Results} -> + handle_batch_result(Results, 0); + Result -> + Result + end + end; + _ -> + Log = #{ + connector => InstId, + request => BatchReq, + state => State, + msg => "invalid request" + }, + ?SLOG(error, Log), + {error, {unrecoverable_error, invalid_request}} + end. + +proc_sql_params(query, SQLOrKey, Params, _State) -> + {SQLOrKey, Params}; +proc_sql_params(TypeOrKey, SQLOrData, Params, #{ + params_tokens := ParamsTokens, prepare_sql := PrepareSql +}) -> + Key = to_bin(TypeOrKey), + case maps:get(Key, ParamsTokens, undefined) of + undefined -> + {SQLOrData, Params}; + Tokens -> + case maps:get(Key, PrepareSql, undefined) of + undefined -> + {SQLOrData, Params}; + Sql -> + {Sql, emqx_plugin_libs_rule:proc_sql(Tokens, SQLOrData)} + end + end. + +on_sql_query(InstId, PoolName, Type, ApplyMode, NameOrSQL, Data) -> + case ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, ApplyMode) of + {error, Reason} = Result -> + ?tp( + oracle_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "oracle database connector do sql query failed", + connector => InstId, + type => Type, + sql => NameOrSQL, + reason => Reason + }), + Result; + Result -> + ?tp( + oracle_connector_query_return, + #{result => Result} + ), + Result + end. + +on_get_status(_InstId, #{pool_name := Pool} = State) -> + case emqx_resource_pool:health_check_workers(Pool, fun ?MODULE:do_get_status/1) of + true -> + case do_check_prepares(State) of + ok -> + connected; + {ok, NState} -> + %% return new state with prepared statements + {connected, NState} + end; + false -> + disconnected + end. + +do_get_status(Conn) -> + ok == element(1, jamdb_oracle:sql_query(Conn, "select 1 from dual")). + +do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> + ok; +do_check_prepares(State = #{pool_name := PoolName, prepare_sql := {error, Prepares}}) -> + {ok, Sts} = prepare_sql(Prepares, PoolName), + {ok, State#{prepare_sql => Prepares, prepare_statement := Sts}}. + +%% =================================================================== + +oracle_host_options() -> + ?ORACLE_HOST_OPTIONS. + +connect(Opts) -> + Password = emqx_secret:unwrap(proplists:get_value(password, Opts)), + NewOpts = lists:keyreplace(password, 1, Opts, {password, Password}), + jamdb_oracle:start_link(NewOpts). + +sql_query_to_str(SqlQuery) -> + emqx_plugin_libs_rule:str(SqlQuery). + +sql_params_to_str(Params) when is_list(Params) -> + lists:map( + fun + (false) -> "0"; + (true) -> "1"; + (Value) -> emqx_plugin_libs_rule:str(Value) + end, + Params + ). + +query(Conn, SQL, Params) -> + Ret = jamdb_oracle:sql_query(Conn, {sql_query_to_str(SQL), sql_params_to_str(Params)}), + ?tp(oracle_query, #{conn => Conn, sql => SQL, params => Params, result => Ret}), + handle_result(Ret). + +execute_batch(Conn, SQL, ParamsList) -> + ParamsListStr = lists:map(fun sql_params_to_str/1, ParamsList), + Ret = jamdb_oracle:sql_query(Conn, {batch, sql_query_to_str(SQL), ParamsListStr}), + ?tp(oracle_batch_query, #{conn => Conn, sql => SQL, params => ParamsList, result => Ret}), + handle_result(Ret). + +parse_prepare_sql(Config) -> + SQL = + case maps:get(prepare_statement, Config, undefined) of + undefined -> + case maps:get(sql, Config, undefined) of + undefined -> #{}; + Template -> #{<<"send_message">> => Template} + end; + Any -> + Any + end, + parse_prepare_sql(maps:to_list(SQL), #{}, #{}). + +parse_prepare_sql([{Key, H} | T], Prepares, Tokens) -> + {PrepareSQL, ParamsTokens} = emqx_plugin_libs_rule:preproc_sql(H, ':n'), + parse_prepare_sql( + T, Prepares#{Key => PrepareSQL}, Tokens#{Key => ParamsTokens} + ); +parse_prepare_sql([], Prepares, Tokens) -> + #{ + prepare_sql => Prepares, + params_tokens => Tokens + }. + +init_prepare(State = #{prepare_sql := Prepares, pool_name := PoolName}) -> + {ok, Sts} = prepare_sql(Prepares, PoolName), + State#{prepare_statement := Sts}. + +prepare_sql(Prepares, PoolName) when is_map(Prepares) -> + prepare_sql(maps:to_list(Prepares), PoolName); +prepare_sql(Prepares, PoolName) -> + Data = do_prepare_sql(Prepares, PoolName), + {ok, _Sts} = Data, + ecpool:add_reconnect_callback(PoolName, {?MODULE, prepare_sql_to_conn, [Prepares]}), + Data. + +do_prepare_sql(Prepares, PoolName) -> + do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + +do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> + {ok, Conn} = ecpool_worker:client(Worker), + {ok, Sts} = prepare_sql_to_conn(Conn, Prepares), + do_prepare_sql(T, Prepares, PoolName, Sts); +do_prepare_sql([], _Prepares, _PoolName, LastSts) -> + {ok, LastSts}. + +prepare_sql_to_conn(Conn, Prepares) -> + prepare_sql_to_conn(Conn, Prepares, #{}). + +prepare_sql_to_conn(Conn, [], Statements) when is_pid(Conn) -> {ok, Statements}; +prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList], Statements) when is_pid(Conn) -> + LogMeta = #{msg => "Oracle Database Prepare Statement", name => Key, prepare_sql => SQL}, + ?SLOG(info, LogMeta), + prepare_sql_to_conn(Conn, PrepareList, Statements#{Key => SQL}). + +to_bin(Bin) when is_binary(Bin) -> + Bin; +to_bin(Atom) when is_atom(Atom) -> + erlang:atom_to_binary(Atom). + +handle_result({error, disconnected}) -> + {error, {recoverable_error, disconnected}}; +handle_result({error, Error}) -> + {error, {unrecoverable_error, Error}}; +handle_result({error, socket, closed} = Error) -> + {error, {recoverable_error, Error}}; +handle_result({error, Type, Reason}) -> + {error, {unrecoverable_error, {Type, Reason}}}; +handle_result(Res) -> + Res. + +handle_batch_result([{affected_rows, RowCount} | Rest], Acc) -> + handle_batch_result(Rest, Acc + RowCount); +handle_batch_result([{proc_result, RetCode, _Rows} | Rest], Acc) when RetCode =:= 0 -> + handle_batch_result(Rest, Acc); +handle_batch_result([{proc_result, RetCode, Reason} | _Rest], _Acc) -> + {error, {unrecoverable_error, {RetCode, Reason}}}; +handle_batch_result([], Acc) -> + {ok, Acc}. + +do_async_reply(Result, {ReplyFun, [Context]}) -> + ReplyFun(Context, Result). diff --git a/apps/emqx_oracle/src/emqx_oracle_schema.erl b/apps/emqx_oracle/src/emqx_oracle_schema.erl new file mode 100644 index 000000000..cfa74054a --- /dev/null +++ b/apps/emqx_oracle/src/emqx_oracle_schema.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_oracle_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-define(REF_MODULE, emqx_oracle). + +%% Hocon config schema exports +-export([ + roots/0, + fields/1 +]). + +roots() -> + [{config, #{type => hoconsc:ref(?REF_MODULE, config)}}]. + +fields(config) -> + [{server, server()}, {sid, fun sid/1}] ++ + emqx_connector_schema_lib:relational_db_fields() ++ + emqx_connector_schema_lib:prepare_statement_fields(). + +server() -> + Meta = #{desc => ?DESC(?REF_MODULE, "server")}, + emqx_schema:servers_sc(Meta, (?REF_MODULE):oracle_host_options()). + +sid(type) -> binary(); +sid(desc) -> ?DESC(?REF_MODULE, "sid"); +sid(required) -> true; +sid(_) -> undefined. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index 18ef9e8fb..dcd666f5b 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -69,7 +69,7 @@ -type preproc_sql_opts() :: #{ placeholders => list(binary()), - replace_with => '?' | '$n', + replace_with => '?' | '$n' | ':n', strip_double_quote => boolean() }. @@ -149,7 +149,7 @@ proc_cmd(Tokens, Data, Opts) -> preproc_sql(Sql) -> preproc_sql(Sql, '?'). --spec preproc_sql(binary(), '?' | '$n' | preproc_sql_opts()) -> +-spec preproc_sql(binary(), '?' | '$n' | ':n' | preproc_sql_opts()) -> {prepare_statement_key(), tmpl_token()}. preproc_sql(Sql, ReplaceWith) when is_atom(ReplaceWith) -> preproc_sql(Sql, #{replace_with => ReplaceWith}); @@ -316,13 +316,17 @@ preproc_tmpl_deep_map_key(Key, _) -> replace_with(Tmpl, RE, '?') -> re:replace(Tmpl, RE, "?", [{return, binary}, global]); replace_with(Tmpl, RE, '$n') -> + replace_with(Tmpl, RE, <<"$">>); +replace_with(Tmpl, RE, ':n') -> + replace_with(Tmpl, RE, <<":">>); +replace_with(Tmpl, RE, String) when is_binary(String) -> Parts = re:split(Tmpl, RE, [{return, binary}, trim, group]), {Res, _} = lists:foldl( fun ([Tkn, _Phld], {Acc, Seq}) -> Seq1 = erlang:integer_to_binary(Seq), - {<>, Seq + 1}; + {<>, Seq + 1}; ([Tkn], {Acc, Seq}) -> {<>, Seq} end, @@ -330,6 +334,7 @@ replace_with(Tmpl, RE, '$n') -> Parts ), Res. + parse_nested(<<".", R/binary>>) -> %% ignore the root . parse_nested(R); diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 24b5a3240..bfd7e68fa 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.9"}, + {vsn, "4.3.10"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 8844fe586..9a4c01a2b 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -105,9 +105,8 @@ proc_cmd(Tokens, Data, Opts) -> preproc_sql(Sql) -> emqx_placeholder:preproc_sql(Sql). --spec preproc_sql(Sql :: binary(), ReplaceWith :: '?' | '$n') -> +-spec preproc_sql(Sql :: binary(), ReplaceWith :: '?' | '$n' | ':n') -> {prepare_statement_key(), tmpl_token()}. - preproc_sql(Sql, ReplaceWith) -> emqx_placeholder:preproc_sql(Sql, ReplaceWith). diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 2553e6dd8..3e264cb3e 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/changes/ee/feat-10498.en.md b/changes/ee/feat-10498.en.md new file mode 100644 index 000000000..7222f8957 --- /dev/null +++ b/changes/ee/feat-10498.en.md @@ -0,0 +1 @@ +Implement Oracle Database Bridge, which supports publishing messages to Oracle Database from MQTT topics. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 5544825f8..825175038 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.11"}, + {vsn, "0.1.12"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 38f471ca2..3baf056ec 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -37,7 +37,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_rocketmq, Method), ref(emqx_ee_bridge_sqlserver, Method), ref(emqx_bridge_opents, Method), - ref(emqx_bridge_pulsar, Method ++ "_producer") + ref(emqx_bridge_pulsar, Method ++ "_producer"), + ref(emqx_bridge_oracle, Method) ]. schema_modules() -> @@ -59,7 +60,8 @@ schema_modules() -> emqx_ee_bridge_rocketmq, emqx_ee_bridge_sqlserver, emqx_bridge_opents, - emqx_bridge_pulsar + emqx_bridge_pulsar, + emqx_bridge_oracle ]. examples(Method) -> @@ -100,7 +102,8 @@ resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; resource_type(sqlserver) -> emqx_ee_connector_sqlserver; resource_type(opents) -> emqx_bridge_opents_connector; -resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer. +resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer; +resource_type(oracle) -> emqx_oracle. fields(bridges) -> [ @@ -167,6 +170,14 @@ fields(bridges) -> desc => <<"OpenTSDB Bridge Config">>, required => false } + )}, + {oracle, + mk( + hoconsc:map(name, ref(emqx_bridge_oracle, "config")), + #{ + desc => <<"Oracle Bridge Config">>, + required => false + } )} ] ++ kafka_structs() ++ pulsar_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ diff --git a/mix.exs b/mix.exs index 2c391611e..8e100967b 100644 --- a/mix.exs +++ b/mix.exs @@ -170,7 +170,9 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_rocketmq, :emqx_bridge_tdengine, :emqx_bridge_timescale, - :emqx_bridge_pulsar + :emqx_bridge_pulsar, + :emqx_oracle, + :emqx_bridge_oracle ]) end @@ -377,6 +379,8 @@ defmodule EMQXUmbrella.MixProject do emqx_bridge_rocketmq: :permanent, emqx_bridge_tdengine: :permanent, emqx_bridge_timescale: :permanent, + emqx_oracle: :permanent, + emqx_bridge_oracle: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rebar.config.erl b/rebar.config.erl index fa7bdbdf3..bcc104b31 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -94,6 +94,8 @@ is_community_umbrella_app("apps/emqx_bridge_redis") -> false; is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false; is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false; is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; +is_community_umbrella_app("apps/emqx_bridge_oracle") -> false; +is_community_umbrella_app("apps/emqx_oracle") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -470,6 +472,8 @@ relx_apps_per_edition(ee) -> emqx_bridge_rocketmq, emqx_bridge_tdengine, emqx_bridge_timescale, + emqx_oracle, + emqx_bridge_oracle, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_bridge_oracle.hocon b/rel/i18n/emqx_bridge_oracle.hocon new file mode 100644 index 000000000..95e0cf4af --- /dev/null +++ b/rel/i18n/emqx_bridge_oracle.hocon @@ -0,0 +1,52 @@ +emqx_bridge_oracle { + + local_topic { + desc = "The MQTT topic filter to be forwarded to Oracle Database. All MQTT 'PUBLISH' messages with the topic" + " matching the local_topic will be forwarded.
" + "NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is" + " configured, then both the data got from the rule and the MQTT messages that match local_topic" + " will be forwarded." + label = "Local Topic" + } + + sql_template { + desc = "SQL Template. The template string can contain placeholders" + " for message metadata and payload field. The placeholders are inserted" + " without any checking and special formatting, so it is important to" + " ensure that the inserted values are formatted and escaped correctly." + label = "SQL Template" + } + + server { + desc = "The IPv4 or IPv6 address or the hostname to connect to.
" + "A host entry has the following form: `Host[:Port]`.
" + "The Oracle Database default port 1521 is used if `[:Port]` is not specified." + label = "Server Host" + } + + sid { + desc = "Sid for Oracle Database" + label = "Oracle Database Sid." + } + + config_enable { + desc = "Enable or disable this bridge" + label = "Enable Or Disable Bridge" + } + + desc_config { + desc = "Configuration for an Oracle Database bridge." + label = "Oracle Database Bridge Configuration" + } + + desc_type { + desc = "The Bridge Type" + label = "Bridge Type" + } + + desc_name { + desc = "Bridge name." + label = "Bridge Name" + } + +} diff --git a/rel/i18n/emqx_oracle.hocon b/rel/i18n/emqx_oracle.hocon new file mode 100644 index 000000000..58de8e4c7 --- /dev/null +++ b/rel/i18n/emqx_oracle.hocon @@ -0,0 +1,15 @@ +emqx_oracle { + + server { + desc = "The IPv4 or IPv6 address or the hostname to connect to.
" + "A host entry has the following form: `Host[:Port]`.
" + "The Oracle Database default port 1521 is used if `[:Port]` is not specified." + label = "Server Host" + } + + sid { + desc = "Sid for Oracle Database." + label = "Oracle Database Sid" + } + +} diff --git a/rel/i18n/zh/emqx_bridge_oracle.hocon b/rel/i18n/zh/emqx_bridge_oracle.hocon new file mode 100644 index 000000000..290ac6d07 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_oracle.hocon @@ -0,0 +1,51 @@ +emqx_bridge_oracle { + + local_topic { + desc = "发送到 'local_topic' 的消息都会转发到 Oracle Database。
" + "注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。" + label = "本地 Topic" + } + + sql_template { + desc = "SQL模板。模板字符串可以包含消息元数据和有效载荷字段的占位符。占位符" + "的插入不需要任何检查和特殊格式化，因此必须确保插入的数值格式化和转义正确。模板字符串可以包含占位符" + "模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入" + "所以必须确保插入的值的格式正确。因此，确保插入的值格式化和转义正确是非常重要的。模板字符串可以包含占位符" + "模板字符串可以包含消息元数据和有效载荷字段的占位符。这些占位符被插入" + "所以必须确保插入的值的格式正确。确保插入的值被正确地格式化和转义。" + label = "SQL 模板" + } + + server { + desc = "将要连接的 IPv4 或 IPv6 地址，或者主机名。
" + "主机名具有以下形式：`Host[:Port]`。
" + "如果未指定 `[:Port]`，则使用 Oracle Database 默认端口 1521。" + label = "服务器地址" + } + + sid { + desc = "Oracle Database Sid 名称" + label = "Oracle Database Sid" + } + + config_enable { + desc = "启用/禁用桥接" + label = "启用/禁用桥接" + } + + desc_config { + desc = "Oracle Database 桥接配置" + label = "Oracle Database 桥接配置" + } + + desc_type { + desc = "Bridge 类型" + label = "桥接类型" + } + + desc_name { + desc = "桥接名字" + label = "桥接名字" + } + +} diff --git a/rel/i18n/zh/emqx_oracle.hocon b/rel/i18n/zh/emqx_oracle.hocon new file mode 100644 index 000000000..70c597cb1 --- /dev/null +++ b/rel/i18n/zh/emqx_oracle.hocon @@ -0,0 +1,15 @@ +emqx_oracle { + + server { + desc = "将要连接的 IPv4 或 IPv6 地址，或者主机名。
" + "主机名具有以下形式：`Host[:Port]`。
" + "如果未指定 `[:Port]`，则使用 Oracle Database 默认端口 1521。" + label = "服务器地址" + } + + sid { + desc = "Oracle Database Sid 名称" + label = "Oracle Database Sid" + } + +} diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index fec0d589c..ad0736bb3 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -193,6 +193,9 @@ for dep in ${CT_DEPS}; do ;; pulsar) FILES+=( '.ci/docker-compose-file/docker-compose-pulsar.yaml' ) + ;; + oracle) + FILES+=( '.ci/docker-compose-file/docker-compose-oracle.yaml' ) ;; *) echo "unknown_ct_dependency $dep" From 43bb6f00caaa0adda300a61e8b359c7f49f28d41 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Tue, 25 Apr 2023 19:51:15 -0300 Subject: [PATCH 123/194] fix(oracle): drop support for async queries jamdb_oracle does not provide interface for performing async queries and ecpool does not monitor the worker which calls jamdb_oracle, so it's safer to keep support for sync queries only. --- .../test/emqx_bridge_oracle_SUITE.erl | 80 ------------------- apps/emqx_oracle/src/emqx_oracle.erl | 75 +---------------- 2 files changed, 4 insertions(+), 151 deletions(-) diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index de77b26de..b50788277 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -330,8 +330,6 @@ create_rule_and_action_http(Config) -> %% Testcases %%------------------------------------------------------------------------------ -% Under normal operations, the bridge will be called async via -% `simple_async_query'. t_sync_query(Config) -> ResourceId = resource_id(Config), ?check_trace( @@ -360,48 +358,6 @@ t_sync_query(Config) -> ), ok. -t_async_query(Config) -> - Overrides = #{ - <<"resource_opts">> => #{ - <<"enable_batch">> => <<"false">>, - <<"batch_size">> => 1 - } - }, - ResourceId = resource_id(Config), - ?check_trace( - begin - ?assertMatch({ok, _}, create_bridge_api(Config, Overrides)), - ?retry( - _Sleep = 1_000, - _Attempts = 20, - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) - ), - reset_table(Config), - MsgId = erlang:unique_integer(), - Params = #{ - topic => ?config(mqtt_topic, Config), - id => MsgId, - payload => ?config(oracle_name, Config), - retain => false - }, - Message = {send_message, Params}, - ?assertMatch( - { - ok, - {ok, #{result := {ok, [{affected_rows, 1}]}}} - }, - ?wait_async_action( - emqx_resource:query(ResourceId, Message), - #{?snk_kind := oracle_query}, - 5_000 - ) - ), - ok - end, - [] - ), - ok. - t_batch_sync_query(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), @@ -449,42 +405,6 @@ t_batch_sync_query(Config) -> ), ok. -t_batch_async_query(Config) -> - ResourceId = resource_id(Config), - ?check_trace( - begin - ?assertMatch({ok, _}, create_bridge_api(Config)), - ?retry( - _Sleep = 1_000, - _Attempts = 20, - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) - ), - reset_table(Config), - MsgId = erlang:unique_integer(), - Params = #{ - topic => ?config(mqtt_topic, Config), - id => MsgId, - payload => ?config(oracle_name, Config), - retain => false - }, - Message = {send_message, Params}, - ?assertMatch( - { - ok, - {ok, #{result := {ok, [{affected_rows, 1}]}}} - }, - ?wait_async_action( - emqx_resource:query(ResourceId, Message), - #{?snk_kind := oracle_batch_query}, - 5_000 - ) - ), - ok - end, - [] - ), - ok. - t_create_via_http(Config) -> ?check_trace( begin diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index c39a6a6d7..a0d7169f3 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -23,8 +23,6 @@ on_stop/2, on_query/3, on_batch_query/3, - on_query_async/4, - on_batch_query_async/4, on_get_status/2 ]). @@ -35,7 +33,6 @@ -export([ query/3, execute_batch/3, - do_async_reply/2, do_get_status/1 ]). @@ -46,7 +43,6 @@ -define(ACTION_SEND_MESSAGE, send_message). -define(SYNC_QUERY_MODE, no_handover). --define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). -define(ORACLE_HOST_OPTIONS, #{ default_port => ?ORACLE_DEFAULT_PORT @@ -67,7 +63,10 @@ batch_params_tokens := params_tokens() }. -callback_mode() -> async_if_possible. +% As ecpool is not monitoring the worker's PID when doing a handover_async, the +% request can be lost if worker crashes. Thus, it's better to force requests to +% be sync for now. +callback_mode() -> always_sync. is_buffer_supported() -> false. @@ -147,24 +146,6 @@ on_query( Res = on_sql_query(InstId, PoolName, Type, ?SYNC_QUERY_MODE, NameOrSQL2, Data), handle_result(Res). -on_query_async(InstId, {TypeOrKey, NameOrSQL}, Reply, State) -> - on_query_async(InstId, {TypeOrKey, NameOrSQL, []}, Reply, State); -on_query_async( - InstId, {TypeOrKey, NameOrSQL, Params} = Query, Reply, #{pool_name := PoolName} = State -) -> - ?SLOG(debug, #{ - msg => "oracle database connector received async sql query", - connector => InstId, - query => Query, - reply => Reply, - state => State - }), - ApplyMode = ?ASYNC_QUERY_MODE(Reply), - Type = query, - {NameOrSQL2, Data} = proc_sql_params(TypeOrKey, NameOrSQL, Params, State), - Res = on_sql_query(InstId, PoolName, Type, ApplyMode, NameOrSQL2, Data), - handle_result(Res). - on_batch_query( InstId, BatchReq, @@ -207,51 +188,6 @@ on_batch_query( {error, {unrecoverable_error, invalid_request}} end. -on_batch_query_async( - InstId, - BatchReq, - Reply, - #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State -) -> - case BatchReq of - [{Key, _} = Request | _] -> - BinKey = to_bin(Key), - case maps:get(BinKey, Tokens, undefined) of - undefined -> - Log = #{ - connector => InstId, - first_request => Request, - state => State, - msg => "batch prepare not implemented" - }, - ?SLOG(error, Log), - {error, {unrecoverable_error, batch_prepare_not_implemented}}; - TokenList -> - {_, Datas} = lists:unzip(BatchReq), - Datas2 = [emqx_plugin_libs_rule:proc_sql(TokenList, Data) || Data <- Datas], - St = maps:get(BinKey, Sts), - case - on_sql_query( - InstId, PoolName, execute_batch, ?ASYNC_QUERY_MODE(Reply), St, Datas2 - ) - of - {ok, Results} -> - handle_batch_result(Results, 0); - Result -> - Result - end - end; - _ -> - Log = #{ - connector => InstId, - request => BatchReq, - state => State, - msg => "invalid request" - }, - ?SLOG(error, Log), - {error, {unrecoverable_error, invalid_request}} - end. - proc_sql_params(query, SQLOrKey, Params, _State) -> {SQLOrKey, Params}; proc_sql_params(TypeOrKey, SQLOrData, Params, #{ @@ -429,6 +365,3 @@ handle_batch_result([{proc_result, RetCode, Reason} | _Rest], _Acc) -> {error, {unrecoverable_error, {RetCode, Reason}}}; handle_batch_result([], Acc) -> {ok, Acc}. - -do_async_reply(Result, {ReplyFun, [Context]}) -> - ReplyFun(Context, Result). From d0c4c70f74331b4951bce0fef6de49e1945027ec Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 13:19:26 -0300 Subject: [PATCH 124/194] test(banned): attempt to fix flaky test Example failure: https://github.com/emqx/emqx/actions/runs/4821105856/jobs/8587006829#step:8:4495 ``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - emqx_common_test_helpers:wait_for_down failed on line 434 Reason: {{t_session_taken,178,timeout},[{emqx_common_test_helpers,...},{...}|...]} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Testing lib.emqx.emqx_banned_SUITE: *** FAILED test case 4 of 5 *** %%% emqx_banned_SUITE ==> t_session_taken: FAILED %%% emqx_banned_SUITE ==> {{t_session_taken,178,timeout}, [{emqx_common_test_helpers,wait_for_down,6, [{file,"/__w/emqx/emqx/source/apps/emqx/test/emqx_common_test_helpers.erl"}, {line,434}]}, {emqx_banned_SUITE,t_session_taken,1, [{file,"/__w/emqx/emqx/source/apps/emqx/test/emqx_banned_SUITE.erl"}, {line,176}]}, {test_server,ts_tc,3,[{file,"test_server.erl"},{line,1782}]}, {test_server,run_test_case_eval1,6,[{file,"test_server.erl"},{line,1291}]}, {test_server,run_test_case_eval,9,[{file,"test_server.erl"},{line,1223}]}]} ``` --- apps/emqx/test/emqx_banned_SUITE.erl | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 0c14f64c9..9419ba4c3 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -186,7 +186,7 @@ t_session_taken(_) -> false end end, - 6000 + 15_000 ), Publish(), diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 2553e6dd8..3e264cb3e 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ From d845c4807d2499637edb1ea13fea94fa90b8be94 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 11:38:30 -0300 Subject: [PATCH 125/194] fix(ocsp): disable periodic refresh when listener or stapling are disabled Fixes https://emqx.atlassian.net/browse/EMQX-9773 --- apps/emqx/src/emqx_listeners.erl | 31 ++++++++-- apps/emqx/src/emqx_ocsp_cache.erl | 16 +++++ apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 77 +++++++++++++++++++++++- 3 files changed, 117 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index f82aebe7c..18ddcaba2 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -441,6 +441,7 @@ post_config_update([listeners, Type, Name], {create, _Request}, NewConf, undefin start_listener(Type, Name, NewConf); post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) -> try_clear_ssl_files(certs_dir(Type, Name), NewConf, OldConf), + ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), case NewConf of #{enabled := true} -> restart_listener(Type, Name, {OldConf, NewConf}); _ -> ok @@ -448,6 +449,7 @@ post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf post_config_update([listeners, _Type, _Name], '$remove', undefined, undefined, _AppEnvs) -> ok; post_config_update([listeners, Type, Name], '$remove', undefined, OldConf, _AppEnvs) -> + ok = unregister_ocsp_stapling_refresh(Type, Name), case stop_listener(Type, Name, OldConf) of ok -> _ = emqx_authentication:delete_chain(listener_id(Type, Name)), @@ -460,10 +462,18 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo #{enabled := NewEnabled} = NewConf, #{enabled := OldEnabled} = OldConf, case {NewEnabled, OldEnabled} of - {true, true} -> restart_listener(Type, Name, {OldConf, NewConf}); - {true, false} -> start_listener(Type, Name, NewConf); - {false, true} -> stop_listener(Type, Name, OldConf); - {false, false} -> stop_listener(Type, Name, OldConf) + {true, true} -> + ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), + restart_listener(Type, Name, {OldConf, NewConf}); + {true, false} -> + ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), + start_listener(Type, Name, NewConf); + {false, true} -> + ok = unregister_ocsp_stapling_refresh(Type, Name), + stop_listener(Type, Name, OldConf); + {false, false} -> + ok = unregister_ocsp_stapling_refresh(Type, Name), + stop_listener(Type, Name, OldConf) end; post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) -> ok. @@ -813,3 +823,16 @@ inject_crl_config( }; inject_crl_config(Conf) -> Conf. + +maybe_unregister_ocsp_stapling_refresh( + ssl = Type, Name, #{ssl_options := #{ocsp := #{enable_ocsp_stapling := false}}} = _Conf +) -> + unregister_ocsp_stapling_refresh(Type, Name), + ok; +maybe_unregister_ocsp_stapling_refresh(_Type, _Name, _Conf) -> + ok. + +unregister_ocsp_stapling_refresh(Type, Name) -> + ListenerId = listener_id(Type, Name), + emqx_ocsp_cache:unregister_listener(ListenerId), + ok. diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 3bb10ee5c..ef0411b37 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -30,6 +30,7 @@ sni_fun/2, fetch_response/1, register_listener/2, + unregister_listener/1, inject_sni_fun/2 ]). @@ -107,6 +108,9 @@ fetch_response(ListenerID) -> register_listener(ListenerID, Opts) -> gen_server:call(?MODULE, {register_listener, ListenerID, Opts}, ?CALL_TIMEOUT). +unregister_listener(ListenerID) -> + gen_server:cast(?MODULE, {unregister_listener, ListenerID}). + -spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). inject_sni_fun(ListenerID, Conf0) -> SNIFun = emqx_const_v1:make_sni_fun(ListenerID), @@ -160,6 +164,18 @@ handle_call({register_listener, ListenerID, Conf}, _From, State0) -> handle_call(Call, _From, State) -> {reply, {error, {unknown_call, Call}}, State}. +handle_cast({unregister_listener, ListenerID}, State0) -> + State2 = + case maps:take(?REFRESH_TIMER(ListenerID), State0) of + error -> + State0; + {TRef, State1} -> + emqx_utils:cancel_timer(TRef), + State1 + end, + State = maps:remove({refresh_interval, ListenerID}, State2), + ?tp(ocsp_cache_listener_unregistered, #{listener_id => ListenerID}), + {noreply, State}; handle_cast(_Cast, State) -> {noreply, State}. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 15ca29853..75c41b9fb 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -254,10 +254,15 @@ does_module_exist(Mod) -> end. assert_no_http_get() -> + Timeout = 0, + Error = should_be_cached, + assert_no_http_get(Timeout, Error). + +assert_no_http_get(Timeout, Error) -> receive {http_get, _URL} -> - error(should_be_cached) - after 0 -> + error(Error) + after Timeout -> ok end. @@ -702,7 +707,9 @@ do_t_update_listener(Config) -> %% the API converts that to an internally %% managed file <<"issuer_pem">> => IssuerPem, - <<"responder_url">> => <<"http://localhost:9877">> + <<"responder_url">> => <<"http://localhost:9877">>, + %% for quicker testing; min refresh in tests is 5 s. + <<"refresh_interval">> => <<"5s">> } } }, @@ -739,6 +746,70 @@ do_t_update_listener(Config) -> ) ), assert_http_get(1, 5_000), + + %% Disable OCSP Stapling; the periodic refreshes should stop + RefreshInterval = emqx_config:get([listeners, ssl, default, ssl_options, ocsp, refresh_interval]), + OCSPConfig1 = + #{ + <<"ssl_options">> => + #{ + <<"ocsp">> => + #{ + <<"enable_ocsp_stapling">> => false + } + } + }, + ListenerData3 = emqx_utils_maps:deep_merge(ListenerData2, OCSPConfig1), + {ok, {_, _, ListenerData4}} = update_listener_via_api(ListenerId, ListenerData3), + ?assertMatch( + #{ + <<"ssl_options">> := + #{ + <<"ocsp">> := + #{ + <<"enable_ocsp_stapling">> := false + } + } + }, + ListenerData4 + ), + + assert_no_http_get(2 * RefreshInterval, should_stop_refreshing), + + ok. + +t_double_unregister(_Config) -> + ListenerID = <<"ssl:test_ocsp">>, + Conf = emqx_config:get_listener_conf(ssl, test_ocsp, []), + ?check_trace( + begin + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:register_listener(ListenerID, Conf), + #{?snk_kind := ocsp_http_fetch_and_cache, listener_id := ListenerID}, + 5_000 + ), + assert_http_get(1), + + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:unregister_listener(ListenerID), + #{?snk_kind := ocsp_cache_listener_unregistered, listener_id := ListenerID}, + 5_000 + ), + + %% Should be idempotent and not crash + {ok, {ok, _}} = + ?wait_async_action( + emqx_ocsp_cache:unregister_listener(ListenerID), + #{?snk_kind := ocsp_cache_listener_unregistered, listener_id := ListenerID}, + 5_000 + ), + ok + end, + [] + ), + ok. t_ocsp_responder_error_responses(_Config) -> From 77f5e461a304cdcefc534e95baff2539f41c99c6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 09:24:35 -0300 Subject: [PATCH 126/194] chore: bump ehttpc -> 0.4.8 Fixes https://emqx.atlassian.net/browse/EMQX-9656 See also https://github.com/emqx/ehttpc/pull/45 This fixes a race condition where the remote server would close the connection before or during requests, and, depending on timing, an `{error, normal}` response would be returned. In those cases, we should just retry the request without using up "retry credits". --- changes/ce/fix-10548.en.md | 2 ++ mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-10548.en.md diff --git a/changes/ce/fix-10548.en.md b/changes/ce/fix-10548.en.md new file mode 100644 index 000000000..d96f0b57f --- /dev/null +++ b/changes/ce/fix-10548.en.md @@ -0,0 +1,2 @@ +Fixed a race condition in the HTTP driver that would result in an error rather than a retry of the request. +Related fix in the driver: https://github.com/emqx/ehttpc/pull/45 diff --git a/mix.exs b/mix.exs index 2b8de4c54..1d0ce7063 100644 --- a/mix.exs +++ b/mix.exs @@ -49,7 +49,7 @@ defmodule EMQXUmbrella.MixProject do {:redbug, "2.0.8"}, {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, - {:ehttpc, github: "emqx/ehttpc", tag: "0.4.7", override: true}, + {:ehttpc, github: "emqx/ehttpc", tag: "0.4.8", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, diff --git a/rebar.config b/rebar.config index 7e783b56d..d14a8099a 100644 --- a/rebar.config +++ b/rebar.config @@ -56,7 +56,7 @@ , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} - , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}} + , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.8"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} From 6e36139a1743a27a036a757a3554196839531d5d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 22:56:34 +0200 Subject: [PATCH 127/194] chore: bump to e5.0.3-alpha.5 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 9e7e53b32..21eb85dfb 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.3-alpha.4"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.5"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 633eacad3b77d2d0860b75e9fc3d1490882770af Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 27 Apr 2023 09:42:05 -0300 Subject: [PATCH 128/194] test(pulsar): add more test cases for Pulsar Producer bridge Fixes https://emqx.atlassian.net/browse/EMQX-8400 --- apps/emqx_bridge_pulsar/rebar.config | 2 +- .../src/emqx_bridge_pulsar.app.src | 2 +- .../src/emqx_bridge_pulsar_impl_producer.erl | 47 ++-- ...emqx_bridge_pulsar_impl_producer_SUITE.erl | 201 +++++++++++++++++- 4 files changed, 235 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge_pulsar/rebar.config b/apps/emqx_bridge_pulsar/rebar.config index be5f282df..d5a63f320 100644 --- a/apps/emqx_bridge_pulsar/rebar.config +++ b/apps/emqx_bridge_pulsar/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {pulsar, {git, "https://github.com/emqx/pulsar-client-erl.git", {tag, "0.8.0"}}}, + {pulsar, {git, "https://github.com/emqx/pulsar-client-erl.git", {tag, "0.8.1"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index ead7cb715..b169aa2c4 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 2bd44d16a..27d50f077 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -87,11 +87,14 @@ on_start(InstanceId, Config) -> }, case pulsar:ensure_supervised_client(ClientId, Servers, ClientOpts) of {ok, _Pid} -> - ?SLOG(info, #{ - msg => "pulsar_client_started", - instance_id => InstanceId, - pulsar_hosts => Servers - }); + ?tp( + info, + "pulsar_client_started", + #{ + instance_id => InstanceId, + pulsar_hosts => Servers + } + ); {error, Reason} -> ?SLOG(error, #{ msg => "failed_to_start_pulsar_client", @@ -115,7 +118,7 @@ on_stop(_InstanceId, State) -> ok. -spec on_get_status(manager_id(), state()) -> connected | disconnected. -on_get_status(_InstanceId, State) -> +on_get_status(_InstanceId, State = #{}) -> #{ pulsar_client_id := ClientId, producers := Producers @@ -135,7 +138,11 @@ on_get_status(_InstanceId, State) -> end; {error, _} -> disconnected - end. + end; +on_get_status(_InstanceId, _State) -> + %% If a health check happens just after a concurrent request to + %% create the bridge is not quite finished, `State = undefined'. + connecting. -spec on_query(manager_id(), {send_message, map()}, state()) -> {ok, term()} @@ -160,6 +167,13 @@ on_query(_InstanceId, {send_message, Message}, State) -> ) -> {ok, pid()}. on_query_async(_InstanceId, {send_message, Message}, AsyncReplyFn, State) -> + ?tp_span( + pulsar_producer_on_query_async, + #{instance_id => _InstanceId, message => Message}, + do_on_query_async(Message, AsyncReplyFn, State) + ). + +do_on_query_async(Message, AsyncReplyFn, State) -> #{ producers := Producers, message_template := MessageTemplate @@ -283,6 +297,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> drop_if_highmem => MemOLP }, ProducerName = producer_name(ClientId), + ?tp(pulsar_producer_capture_name, #{producer_name => ProducerName}), MessageTemplate = compile_message_template(MessageTemplateOpts), ProducerOpts0 = #{ @@ -298,6 +313,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> }, ProducerOpts = maps:merge(ReplayQOpts, ProducerOpts0), PulsarTopic = binary_to_list(PulsarTopic0), + ?tp(pulsar_producer_about_to_start_producers, #{producer_name => ProducerName}), try pulsar:ensure_supervised_producers(ClientId, PulsarTopic, ProducerOpts) of {ok, Producers} -> State = #{ @@ -310,13 +326,16 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) -> {ok, State} catch Kind:Error:Stacktrace -> - ?SLOG(error, #{ - msg => "failed_to_start_pulsar_producer", - instance_id => InstanceId, - kind => Kind, - reason => Error, - stacktrace => Stacktrace - }), + ?tp( + error, + "failed_to_start_pulsar_producer", + #{ + instance_id => InstanceId, + kind => Kind, + reason => Error, + stacktrace => Stacktrace + } + ), stop_client(ClientId), throw(failed_to_start_pulsar_producer) end. diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index d254b01fc..be38f6625 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -37,7 +37,14 @@ groups() -> ]. only_once_tests() -> - [t_create_via_http]. + [ + t_create_via_http, + t_start_when_down, + t_send_when_down, + t_send_when_timeout, + t_failure_to_start_producer, + t_producer_process_crash + ]. init_per_suite(Config) -> Config. @@ -753,6 +760,198 @@ t_on_get_status(Config) -> ), ok. +t_start_when_down(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ResourceId = resource_id(Config), + ?check_trace( + begin + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ok + end), + %% Should recover given enough time. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + ok + end, + [] + ), + ok. + +t_send_when_down(Config) -> + do_t_send_with_failure(Config, down). + +t_send_when_timeout(Config) -> + do_t_send_with_failure(Config, timeout). + +do_t_send_with_failure(Config, FailureType) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + MQTTTopic = ?config(mqtt_topic, Config), + QoS = 0, + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), + + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := pulsar_producer_bridge_started}, + 10_000 + ), + ?check_trace( + begin + emqx_common_test_helpers:with_failure( + FailureType, ProxyName, ProxyHost, ProxyPort, fun() -> + {_, {ok, _}} = + ?wait_async_action( + emqx:publish(Message0), + #{ + ?snk_kind := pulsar_producer_on_query_async, + ?snk_span := {complete, _} + }, + 5_000 + ), + ok + end + ), + ok + end, + fun(_Trace) -> + %% Should recover given enough time. + Data0 = receive_consumed(20_000), + ?assertMatch( + [ + #{ + <<"clientid">> := ClientId, + <<"event">> := <<"message.publish">>, + <<"payload">> := Payload, + <<"topic">> := MQTTTopic + } + ], + Data0 + ), + ok + end + ), + ok. + +%% Check that we correctly terminate the pulsar client when the pulsar +%% producer processes fail to start for whatever reason. +t_failure_to_start_producer(Config) -> + ?check_trace( + begin + ?force_ordering( + #{?snk_kind := name_registered}, + #{?snk_kind := pulsar_producer_about_to_start_producers} + ), + spawn_link(fun() -> + ?tp(will_register_name, #{}), + {ok, #{producer_name := ProducerName}} = ?block_until( + #{?snk_kind := pulsar_producer_capture_name}, 10_000 + ), + true = register(ProducerName, self()), + ?tp(name_registered, #{name => ProducerName}), + %% Just simulating another process so that starting the + %% producers fail. Currently it does a gen_server:call + %% with `infinity' timeout, so this is just to avoid + %% hanging. + receive + {'$gen_call', From, _Request} -> + gen_server:reply(From, {error, im_not, your_producer}) + end, + receive + die -> ok + end + end), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := pulsar_bridge_client_stopped}, + 20_000 + ), + ok + end, + [] + ), + ok. + +%% Check the driver recovers itself if one of the producer processes +%% die for whatever reason. +t_producer_process_crash(Config) -> + MQTTTopic = ?config(mqtt_topic, Config), + ResourceId = resource_id(Config), + QoS = 0, + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), + ?check_trace( + begin + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge( + Config, + #{<<"buffer">> => #{<<"mode">> => <<"disk">>}} + ), + #{?snk_kind := pulsar_producer_bridge_started}, + 10_000 + ), + [ProducerPid | _] = [ + Pid + || {_Name, PS, _Type, _Mods} <- supervisor:which_children(pulsar_producers_sup), + Pid <- element(2, process_info(PS, links)), + case proc_lib:initial_call(Pid) of + {pulsar_producer, init, _} -> true; + _ -> false + end + ], + Ref = monitor(process, ProducerPid), + exit(ProducerPid, kill), + receive + {'DOWN', Ref, process, ProducerPid, _Killed} -> + ok + after 1_000 -> ct:fail("pid didn't die") + end, + ?assertEqual({ok, connecting}, emqx_resource_manager:health_check(ResourceId)), + %% Should recover given enough time. + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ), + {_, {ok, _}} = + ?wait_async_action( + emqx:publish(Message0), + #{?snk_kind := pulsar_producer_on_query_async, ?snk_span := {complete, _}}, + 5_000 + ), + Data0 = receive_consumed(20_000), + ?assertMatch( + [ + #{ + <<"clientid">> := ClientId, + <<"event">> := <<"message.publish">>, + <<"payload">> := Payload, + <<"topic">> := MQTTTopic + } + ], + Data0 + ), + ok + end, + [] + ), + ok. + t_cluster(Config) -> MQTTTopic = ?config(mqtt_topic, Config), ResourceId = resource_id(Config), From d3a26b45beb011a64c0b47140353730a73f0c3a0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 12:21:13 +0200 Subject: [PATCH 129/194] docs: update config note --- .gitignore | 2 ++ apps/emqx_conf/etc/emqx_conf.conf | 15 ++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 6c4de9272..ceb12182f 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,5 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi lux_logs/ /.prepare bom.json +ct_run*/ +apps/emqx_conf/etc/emqx.conf.all.rendered* diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 86147bf25..4c03c10c6 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -1,12 +1,13 @@ ## NOTE: -## Configs in this file might be overridden by: -## 1. Environment variables which start with 'EMQX_' prefix -## 2. File $EMQX_NODE__DATA_DIR/configs/cluster-override.conf -## 3. File $EMQX_NODE__DATA_DIR/configs/local-override.conf +## This config file overrides data/configs/cluster.hocon, +## and is merged with environment variables which start with 'EMQX_' prefix. ## -## The *-override.conf files are overwritten at runtime when changes -## are made from EMQX dashboard UI, management HTTP API, or CLI. -## All configuration details can be found in emqx.conf.example +## Config changes made from EMQX dashboard UI, management HTTP API, or CLI +## are stored in data/configs/cluster.hocon. +## To avoid confusion, please do not store the same configs in both files. +## +## See https://docs.emqx.com/en/enterprise/v5.0/configuration/configuration.html +## Configuration full example can be found in emqx.conf.example node { name = "emqx@127.0.0.1" From 7a81b96be0380789d16f75294f0d60a242e1ff94 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 20:48:53 +0200 Subject: [PATCH 130/194] fix(emqx_conf_app): print init_load failure to standard_error logger may not get the chance to spit out the logs before the vm dies, no matter how long sleep is added before init:stop(1) --- apps/emqx_conf/src/emqx_conf_app.erl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index fbfb97a79..dedb9aeab 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -32,12 +32,8 @@ start(_StartType, _StartArgs) -> ok = init_conf() catch C:E:St -> - ?SLOG(critical, #{ - msg => failed_to_init_config, - exception => C, - reason => E, - stacktrace => St - }), + %% logger is not quite ready. + io:format(standard_error, "Failed to load config~n~p~n~p~n~p~n", [C, E, St]), init:stop(1) end, ok = emqx_config_logger:refresh_config(), From ee61648368f65b52bac3c6d2d7de2e36fddcd4fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 22:42:27 +0200 Subject: [PATCH 131/194] build: imporove speed to local run prior to this change, 'make run' has to wait for the release tar ball to be created. now it just copy the release files and run --- build | 9 +++++---- rebar.config | 2 +- rebar.config.erl | 5 +---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/build b/build index 05246a359..eb75317cc 100755 --- a/build +++ b/build @@ -125,6 +125,7 @@ assert_no_compile_time_only_deps() { } make_rel() { + local release_or_tar="${1}" ./scripts/pre-compile.sh "$PROFILE" # make_elixir_rel always create rebar.lock # delete it to make git clone + checkout work because we use shallow close for rebar deps @@ -134,7 +135,7 @@ make_rel() { # generate docs (require beam compiled), generated to etc and priv dirs make_docs # now assemble the release tar - ./rebar3 as "$PROFILE" tar + ./rebar3 as "$PROFILE" "$release_or_tar" assert_no_compile_time_only_deps } @@ -220,7 +221,7 @@ make_tgz() { else # build the src_tarball again to ensure relup is included # elixir does not have relup yet. - make_rel + make_rel tar local relpath="_build/${PROFILE}/rel/emqx" full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)" @@ -378,7 +379,7 @@ case "$ARTIFACT" in make_docs ;; rel) - make_rel + make_rel release ;; relup) make_relup @@ -397,7 +398,7 @@ case "$ARTIFACT" in if [ "${IS_ELIXIR:-}" = 'yes' ]; then make_elixir_rel else - make_rel + make_rel tar fi env EMQX_REL="$(pwd)" \ EMQX_BUILD="${PROFILE}" \ diff --git a/rebar.config b/rebar.config index bd8c3484c..05e9b9a28 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ emqx_ssl_crl_cache ]}. -{provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. +%{provider_hooks, [{pre, [{release, {relup_helper, gen_appups}}]}]}. {post_hooks,[]}. diff --git a/rebar.config.erl b/rebar.config.erl index bcc104b31..5c83d1ea0 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -156,7 +156,7 @@ project_app_dirs(Edition) -> plugins() -> [ - {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}, + %{relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.1.0"}}}, %% emqx main project does not require port-compiler %% pin at root level for deterministic {pc, "v1.14.0"} @@ -495,11 +495,8 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/emqx_cluster_rescue", "bin/emqx_cluster_rescue"}, {copy, "bin/node_dump", "bin/node_dump"}, {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}, - %% for relup {copy, "bin/emqx", "bin/emqx-{{release_version}}"}, - %% for relup {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, - %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, {copy, "apps/emqx_gateway_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, From a19621e533e015897f07d204379ffb293c858fc5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 28 Apr 2023 14:43:21 -0300 Subject: [PATCH 132/194] fix(webhook): treat `{shutdown, normal}` and `{closed, _}` async reply as retriable Apparently, the async reply returned by ehttpc can be `{shutdown, normal}` or `{closed, "The connection was lost."}`, in which case the request should be retried. ``` Apr 28 17:40:41 emqx-0.int.thales bash[48880]: 17:40:41.803 [error] [id: "bridge:webhook:webhook", msg: :unrecoverable_error, reason: {:shutdown, :normal}] Apr 28 18:36:37 emqx-0.int.thales bash[53368]: 18:36:37.605 [error] [id: "bridge:webhook:webhook", msg: :unrecoverable_error, reason: {:closed, 'The connection was lost.'}] ``` --- .../src/emqx_connector_http.erl | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index ef2e11eb7..c89285727 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -306,7 +306,20 @@ on_query( Retry ) of - {error, Reason} when Reason =:= econnrefused; Reason =:= timeout -> + {error, Reason} when + Reason =:= econnrefused; + Reason =:= timeout; + Reason =:= {shutdown, normal}; + Reason =:= {shutdown, closed} + -> + ?SLOG(warning, #{ + msg => "http_connector_do_request_failed", + reason => Reason, + connector => InstId + }), + {error, {recoverable_error, Reason}}; + {error, {closed, _Message} = Reason} -> + %% _Message = "The connection was lost." ?SLOG(warning, #{ msg => "http_connector_do_request_failed", reason => Reason, @@ -568,7 +581,16 @@ reply_delegator(ReplyFunAndArgs, Result) -> case Result of %% The normal reason happens when the HTTP connection times out before %% the request has been fully processed - {error, Reason} when Reason =:= econnrefused; Reason =:= timeout; Reason =:= normal -> + {error, Reason} when + Reason =:= econnrefused; + Reason =:= timeout; + Reason =:= normal; + Reason =:= {shutdown, normal} + -> + Result1 = {error, {recoverable_error, Reason}}, + emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); + {error, {closed, _Message} = Reason} -> + %% _Message = "The connection was lost." Result1 = {error, {recoverable_error, Reason}}, emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result1); _ -> From c58ffce75f61e1dd2045894be47a7d92442aa8a1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 22:49:52 +0200 Subject: [PATCH 133/194] fix(hocon): pin 0.38.2 with the map type value converter fixed --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d954a6b1e..26f58ff69 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 1d0ce7063..93f7417c9 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.38.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.38.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index d14a8099a..267f60ee4 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.2"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From b0f3a654ee864eb5a533f1fb6b009f28d1a62fa6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 12:22:09 +0200 Subject: [PATCH 134/194] refactor: delete default listeners from default config The new config overriding rule is very much confusing for people who wants to persist listener config changes made from dashboard This commit moves the default values from default config file to schema source code. In order to support build-time cert path at runtime, there is also a naive environment variable interplation feature added. --- apps/emqx/etc/emqx.conf | 43 ---------------- apps/emqx/src/emqx_schema.erl | 53 ++++++++++++++++++++ apps/emqx/src/emqx_tls_lib.erl | 91 ++++++++++++++++++++++++++++------ 3 files changed, 129 insertions(+), 58 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index ee345e9d6..e69de29bb 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1,43 +0,0 @@ -listeners.tcp.default { - bind = "0.0.0.0:1883" - max_connections = 1024000 -} - -listeners.ssl.default { - bind = "0.0.0.0:8883" - max_connections = 512000 - ssl_options { - keyfile = "{{ platform_etc_dir }}/certs/key.pem" - certfile = "{{ platform_etc_dir }}/certs/cert.pem" - cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } -} - -listeners.ws.default { - bind = "0.0.0.0:8083" - max_connections = 1024000 - websocket.mqtt_path = "/mqtt" -} - -listeners.wss.default { - bind = "0.0.0.0:8084" - max_connections = 512000 - websocket.mqtt_path = "/mqtt" - ssl_options { - keyfile = "{{ platform_etc_dir }}/certs/key.pem" - certfile = "{{ platform_etc_dir }}/certs/cert.pem" - cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } -} - -# listeners.quic.default { -# enabled = true -# bind = "0.0.0.0:14567" -# max_connections = 1024000 -# ssl_options { -# verify = verify_none -# keyfile = "{{ platform_etc_dir }}/certs/key.pem" -# certfile = "{{ platform_etc_dir }}/certs/cert.pem" -# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" -# } -# } diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ba333f111..c56fc9b48 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -779,6 +779,7 @@ fields("listeners") -> map(name, ref("mqtt_tcp_listener")), #{ desc => ?DESC(fields_listeners_tcp), + default => default_listener(tcp), required => {false, recursively} } )}, @@ -787,6 +788,7 @@ fields("listeners") -> map(name, ref("mqtt_ssl_listener")), #{ desc => ?DESC(fields_listeners_ssl), + default => default_listener(ssl), required => {false, recursively} } )}, @@ -795,6 +797,7 @@ fields("listeners") -> map(name, ref("mqtt_ws_listener")), #{ desc => ?DESC(fields_listeners_ws), + default => default_listener(ws), required => {false, recursively} } )}, @@ -803,6 +806,7 @@ fields("listeners") -> map(name, ref("mqtt_wss_listener")), #{ desc => ?DESC(fields_listeners_wss), + default => default_listener(wss), required => {false, recursively} } )}, @@ -3083,3 +3087,52 @@ assert_required_field(Conf, Key, ErrorMessage) -> _ -> ok end. + +default_listener(tcp) -> + #{ + <<"default">> => + #{ + <<"bind">> => <<"0.0.0.0:1883">>, + <<"max_connections">> => 1024000 + } + }; +default_listener(ws) -> + #{ + <<"default">> => + #{ + <<"bind">> => <<"0.0.0.0:8083">>, + <<"max_connections">> => 1024000, + <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} + } + }; +default_listener(SSLListener) -> + %% The env variable is resolved in emqx_tls_lib + CertFile = fun(Name) -> + iolist_to_binary("${EMQX_ETC_DIR}/" ++ filename:join(["certs", Name])) + end, + SslOptions = #{ + <<"cacertfile">> => CertFile(<<"cacert.pem">>), + <<"certfile">> => CertFile(<<"cert.pem">>), + <<"keyfile">> => CertFile(<<"key.pem">>) + }, + case SSLListener of + ssl -> + #{ + <<"default">> => + #{ + <<"bind">> => <<"0.0.0.0:8883">>, + <<"max_connections">> => 512000, + <<"ssl_options">> => SslOptions + } + }; + wss -> + #{ + <<"default">> => + #{ + <<"bind">> => <<"0.0.0.0:8084">>, + <<"max_connections">> => 512000, + <<"ssl_options">> => SslOptions, + <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} + } + } + end. diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index d1c57bf0d..c555059cf 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -309,19 +309,19 @@ ensure_ssl_files(Dir, SSL, Opts) -> case ensure_ssl_file_key(SSL, RequiredKeys) of ok -> KeyPaths = ?SSL_FILE_OPT_PATHS ++ ?SSL_FILE_OPT_PATHS_A, - ensure_ssl_files(Dir, SSL, KeyPaths, Opts); + ensure_ssl_files_per_key(Dir, SSL, KeyPaths, Opts); {error, _} = Error -> Error end. -ensure_ssl_files(_Dir, SSL, [], _Opts) -> +ensure_ssl_files_per_key(_Dir, SSL, [], _Opts) -> {ok, SSL}; -ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> +ensure_ssl_files_per_key(Dir, SSL, [KeyPath | KeyPaths], Opts) -> case ensure_ssl_file(Dir, KeyPath, SSL, emqx_utils_maps:deep_get(KeyPath, SSL, undefined), Opts) of {ok, NewSSL} -> - ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); + ensure_ssl_files_per_key(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> {error, Reason#{which_options => [KeyPath]}} end. @@ -347,7 +347,8 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> delete_old_file(New, Old) when New =:= Old -> ok; delete_old_file(_New, _Old = undefined) -> ok; -delete_old_file(_New, Old) -> +delete_old_file(_New, Old0) -> + Old = resolve_cert_path(Old0), case is_generated_file(Old) andalso filelib:is_regular(Old) andalso file:delete(Old) of ok -> ok; @@ -355,7 +356,7 @@ delete_old_file(_New, Old) -> false -> ok; {error, Reason} -> - ?SLOG(error, #{msg => "failed_to_delete_ssl_file", file_path => Old, reason => Reason}) + ?SLOG(error, #{msg => "failed_to_delete_ssl_file", file_path => Old0, reason => Reason}) end. ensure_ssl_file(_Dir, _KeyPath, SSL, undefined, _Opts) -> @@ -414,7 +415,8 @@ is_pem(MaybePem) -> %% To make it simple, the file is always overwritten. %% Also a potentially half-written PEM file (e.g. due to power outage) %% can be corrected with an overwrite. -save_pem_file(Dir, KeyPath, Pem, DryRun) -> +save_pem_file(Dir0, KeyPath, Pem, DryRun) -> + Dir = resolve_cert_path(Dir0), Path = pem_file_name(Dir, KeyPath, Pem), case filelib:ensure_dir(Path) of ok when DryRun -> @@ -472,7 +474,8 @@ hex_str(Bin) -> iolist_to_binary([io_lib:format("~2.16.0b", [X]) || <> <= Bin]). %% @doc Returns 'true' when the file is a valid pem, otherwise {error, Reason}. -is_valid_pem_file(Path) -> +is_valid_pem_file(Path0) -> + Path = resolve_cert_path(Path0), case file:read_file(Path) of {ok, Pem} -> is_pem(Pem) orelse {error, not_pem}; {error, Reason} -> {error, Reason} @@ -513,10 +516,15 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> to_server_opts(Type, Opts) -> Versions = integral_versions(Type, maps:get(versions, Opts, undefined)), Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)), - maps:to_list(Opts#{ - ciphers => Ciphers, - versions => Versions - }). + filter( + maps:to_list(Opts#{ + keyfile => resolve_cert_path_strict(maps:get(keyfile, Opts, undefined)), + certfile => resolve_cert_path_strict(maps:get(certfile, Opts, undefined)), + cacertfile => resolve_cert_path_strict(maps:get(cacertfile, Opts, undefined)), + ciphers => Ciphers, + versions => Versions + }) + ). %% @doc Convert hocon-checked tls client options (map()) to %% proplist accepted by ssl library. @@ -532,9 +540,9 @@ to_client_opts(Type, Opts) -> Get = fun(Key) -> GetD(Key, undefined) end, case GetD(enable, false) of true -> - KeyFile = ensure_str(Get(keyfile)), - CertFile = ensure_str(Get(certfile)), - CAFile = ensure_str(Get(cacertfile)), + KeyFile = resolve_cert_path_strict(Get(keyfile)), + CertFile = resolve_cert_path_strict(Get(certfile)), + CAFile = resolve_cert_path_strict(Get(cacertfile)), Verify = GetD(verify, verify_none), SNI = ensure_sni(Get(server_name_indication)), Versions = integral_versions(Type, Get(versions)), @@ -556,6 +564,59 @@ to_client_opts(Type, Opts) -> [] end. +resolve_cert_path_strict(Path) -> + case resolve_cert_path(Path) of + undefined -> + undefined; + ResolvedPath -> + case filelib:is_regular(ResolvedPath) of + true -> + ResolvedPath; + false -> + PathToLog = ensure_str(Path), + LogData = + case PathToLog =:= ResolvedPath of + true -> + #{path => PathToLog}; + false -> + #{path => PathToLog, resolved_path => ResolvedPath} + end, + ?SLOG(error, LogData#{msg => "cert_file_not_found"}), + undefined + end + end. + +resolve_cert_path(undefined) -> + undefined; +resolve_cert_path(Path) -> + case ensure_str(Path) of + "$" ++ Maybe -> + naive_env_resolver(Maybe); + Other -> + Other + end. + +%% resolves a file path like "ENV_VARIABLE/sub/path" or "{ENV_VARIABLE}/sub/path" +%% in windows, it could be "ENV_VARIABLE/sub\path" or "{ENV_VARIABLE}/sub\path" +naive_env_resolver(Maybe) -> + case string:split(Maybe, "/") of + [_] -> + Maybe; + [Env, SubPath] -> + case os:getenv(trim_env_name(Env)) of + false -> + SubPath; + "" -> + SubPath; + EnvValue -> + filename:join(EnvValue, SubPath) + end + end. + +%% delete the first and last curly braces +trim_env_name(Env) -> + string:trim(Env, both, "{}"). + filter([]) -> []; filter([{_, undefined} | T]) -> filter(T); filter([{_, ""} | T]) -> filter(T); From 7c5a9e0e2041a70e09dc9c4101c6bbade062b6e4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 14:20:57 +0200 Subject: [PATCH 135/194] refactor: move the env interpolation function to emqx_schema also added test cases --- apps/emqx/src/emqx_schema.erl | 72 ++++++++++++++++++++----- apps/emqx/src/emqx_tls_lib.erl | 60 ++++++--------------- apps/emqx/test/emqx_schema_tests.erl | 78 ++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 57 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c56fc9b48..8fdd2557e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -66,7 +66,8 @@ user_lookup_fun_tr/2, validate_alarm_actions/1, non_empty_string/1, - validations/0 + validations/0, + naive_env_interpolation/1 ]). -export([qos/0]). @@ -825,7 +826,7 @@ fields("crl_cache") -> %% same URL. If they had diverging timeout options, it would be %% confusing. [ - {"refresh_interval", + {refresh_interval, sc( duration(), #{ @@ -833,7 +834,7 @@ fields("crl_cache") -> desc => ?DESC("crl_cache_refresh_interval") } )}, - {"http_timeout", + {http_timeout, sc( duration(), #{ @@ -841,7 +842,7 @@ fields("crl_cache") -> desc => ?DESC("crl_cache_refresh_http_timeout") } )}, - {"capacity", + {capacity, sc( pos_integer(), #{ @@ -1358,7 +1359,7 @@ fields("ssl_client_opts") -> client_ssl_opts_schema(#{}); fields("ocsp") -> [ - {"enable_ocsp_stapling", + {enable_ocsp_stapling, sc( boolean(), #{ @@ -1366,7 +1367,7 @@ fields("ocsp") -> desc => ?DESC("server_ssl_opts_schema_enable_ocsp_stapling") } )}, - {"responder_url", + {responder_url, sc( url(), #{ @@ -1374,7 +1375,7 @@ fields("ocsp") -> desc => ?DESC("server_ssl_opts_schema_ocsp_responder_url") } )}, - {"issuer_pem", + {issuer_pem, sc( binary(), #{ @@ -1382,7 +1383,7 @@ fields("ocsp") -> desc => ?DESC("server_ssl_opts_schema_ocsp_issuer_pem") } )}, - {"refresh_interval", + {refresh_interval, sc( duration(), #{ @@ -1390,7 +1391,7 @@ fields("ocsp") -> desc => ?DESC("server_ssl_opts_schema_ocsp_refresh_interval") } )}, - {"refresh_http_timeout", + {refresh_http_timeout, sc( duration(), #{ @@ -2317,12 +2318,12 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> Field || not IsRanchListener, Field <- [ - {"gc_after_handshake", + {gc_after_handshake, sc(boolean(), #{ default => false, desc => ?DESC(server_ssl_opts_schema_gc_after_handshake) })}, - {"ocsp", + {ocsp, sc( ref("ocsp"), #{ @@ -2330,7 +2331,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> validator => fun ocsp_inner_validator/1 } )}, - {"enable_crl_check", + {enable_crl_check, sc( boolean(), #{ @@ -3106,7 +3107,7 @@ default_listener(ws) -> } }; default_listener(SSLListener) -> - %% The env variable is resolved in emqx_tls_lib + %% The env variable is resolved in emqx_tls_lib by calling naive_env_interpolate CertFile = fun(Name) -> iolist_to_binary("${EMQX_ETC_DIR}/" ++ filename:join(["certs", Name])) end, @@ -3136,3 +3137,48 @@ default_listener(SSLListener) -> } } end. + +%% @doc This function helps to perform a naive string interpolation which +%% only looks at the first segment of the string and tries to replace it. +%% For example +%% "$MY_FILE_PATH" +%% "${MY_FILE_PATH}" +%% "$ENV_VARIABLE/sub/path" +%% "${ENV_VARIABLE}/sub/path" +%% "${ENV_VARIABLE}\sub\path" # windows +%% This function returns undefined if the input is undefined +%% otherwise always return string. +naive_env_interpolation(undefined) -> + undefined; +naive_env_interpolation(Bin) when is_binary(Bin) -> + naive_env_interpolation(unicode:characters_to_list(Bin, utf8)); +naive_env_interpolation("$" ++ Maybe = Original) -> + {Env, Tail} = split_path(Maybe), + case resolve_env(Env) of + {ok, Path} -> + filename:join([Path, Tail]); + error -> + Original + end; +naive_env_interpolation(Other) -> + Other. + +split_path(Path) -> + split_path(Path, []). + +split_path([], Acc) -> + {lists:reverse(Acc), []}; +split_path([Char | Rest], Acc) when Char =:= $/ orelse Char =:= $\\ -> + {lists:reverse(Acc), string:trim(Rest, leading, "/\\")}; +split_path([Char | Rest], Acc) -> + split_path(Rest, [Char | Acc]). + +resolve_env(Name0) -> + Name = string:trim(Name0, both, "{}"), + Value = os:getenv(Name), + case Value =/= false andalso Value =/= "" of + true -> + {ok, Value}; + false -> + error + end. diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index c555059cf..2683d2a9d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -347,8 +347,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> delete_old_file(New, Old) when New =:= Old -> ok; delete_old_file(_New, _Old = undefined) -> ok; -delete_old_file(_New, Old0) -> - Old = resolve_cert_path(Old0), +delete_old_file(_New, Old) -> case is_generated_file(Old) andalso filelib:is_regular(Old) andalso file:delete(Old) of ok -> ok; @@ -356,7 +355,7 @@ delete_old_file(_New, Old0) -> false -> ok; {error, Reason} -> - ?SLOG(error, #{msg => "failed_to_delete_ssl_file", file_path => Old0, reason => Reason}) + ?SLOG(error, #{msg => "failed_to_delete_ssl_file", file_path => Old, reason => Reason}) end. ensure_ssl_file(_Dir, _KeyPath, SSL, undefined, _Opts) -> @@ -415,8 +414,7 @@ is_pem(MaybePem) -> %% To make it simple, the file is always overwritten. %% Also a potentially half-written PEM file (e.g. due to power outage) %% can be corrected with an overwrite. -save_pem_file(Dir0, KeyPath, Pem, DryRun) -> - Dir = resolve_cert_path(Dir0), +save_pem_file(Dir, KeyPath, Pem, DryRun) -> Path = pem_file_name(Dir, KeyPath, Pem), case filelib:ensure_dir(Path) of ok when DryRun -> @@ -475,7 +473,7 @@ hex_str(Bin) -> %% @doc Returns 'true' when the file is a valid pem, otherwise {error, Reason}. is_valid_pem_file(Path0) -> - Path = resolve_cert_path(Path0), + Path = resolve_cert_path_for_read(Path0), case file:read_file(Path) of {ok, Pem} -> is_pem(Pem) orelse {error, not_pem}; {error, Reason} -> {error, Reason} @@ -516,11 +514,12 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> to_server_opts(Type, Opts) -> Versions = integral_versions(Type, maps:get(versions, Opts, undefined)), Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)), + Path = fun(Key) -> resolve_cert_path_for_read_strict(maps:get(Key, Opts, undefined)) end, filter( maps:to_list(Opts#{ - keyfile => resolve_cert_path_strict(maps:get(keyfile, Opts, undefined)), - certfile => resolve_cert_path_strict(maps:get(certfile, Opts, undefined)), - cacertfile => resolve_cert_path_strict(maps:get(cacertfile, Opts, undefined)), + keyfile => Path(keyfile), + certfile => Path(certfile), + cacertfile => Path(cacertfile), ciphers => Ciphers, versions => Versions }) @@ -538,11 +537,12 @@ to_client_opts(Opts) -> to_client_opts(Type, Opts) -> GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end, Get = fun(Key) -> GetD(Key, undefined) end, + Path = fun(Key) -> resolve_cert_path_for_read_strict(Get(Key)) end, case GetD(enable, false) of true -> - KeyFile = resolve_cert_path_strict(Get(keyfile)), - CertFile = resolve_cert_path_strict(Get(certfile)), - CAFile = resolve_cert_path_strict(Get(cacertfile)), + KeyFile = Path(keyfile), + CertFile = Path(certfile), + CAFile = Path(cacertfile), Verify = GetD(verify, verify_none), SNI = ensure_sni(Get(server_name_indication)), Versions = integral_versions(Type, Get(versions)), @@ -564,8 +564,8 @@ to_client_opts(Type, Opts) -> [] end. -resolve_cert_path_strict(Path) -> - case resolve_cert_path(Path) of +resolve_cert_path_for_read_strict(Path) -> + case resolve_cert_path_for_read(Path) of undefined -> undefined; ResolvedPath -> @@ -586,36 +586,8 @@ resolve_cert_path_strict(Path) -> end end. -resolve_cert_path(undefined) -> - undefined; -resolve_cert_path(Path) -> - case ensure_str(Path) of - "$" ++ Maybe -> - naive_env_resolver(Maybe); - Other -> - Other - end. - -%% resolves a file path like "ENV_VARIABLE/sub/path" or "{ENV_VARIABLE}/sub/path" -%% in windows, it could be "ENV_VARIABLE/sub\path" or "{ENV_VARIABLE}/sub\path" -naive_env_resolver(Maybe) -> - case string:split(Maybe, "/") of - [_] -> - Maybe; - [Env, SubPath] -> - case os:getenv(trim_env_name(Env)) of - false -> - SubPath; - "" -> - SubPath; - EnvValue -> - filename:join(EnvValue, SubPath) - end - end. - -%% delete the first and last curly braces -trim_env_name(Env) -> - string:trim(Env, both, "{}"). +resolve_cert_path_for_read(Path) -> + emqx_schema:naive_env_interpolation(Path). filter([]) -> []; filter([{_, undefined} | T]) -> filter(T); diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 5176f4fad..b4bb85a8a 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -513,3 +513,81 @@ url_type_test_() -> typerefl:from_string(emqx_schema:url(), <<"">>) ) ]. + +env_test_() -> + Do = fun emqx_schema:naive_env_interpolation/1, + [ + {"undefined", fun() -> ?assertEqual(undefined, Do(undefined)) end}, + {"full env abs path", + with_env_fn( + "MY_FILE", + "/path/to/my/file", + fun() -> ?assertEqual("/path/to/my/file", Do("$MY_FILE")) end + )}, + {"full env relative path", + with_env_fn( + "MY_FILE", + "path/to/my/file", + fun() -> ?assertEqual("path/to/my/file", Do("${MY_FILE}")) end + )}, + %% we can not test windows style file join though + {"windows style", + with_env_fn( + "MY_FILE", + "path\\to\\my\\file", + fun() -> ?assertEqual("path\\to\\my\\file", Do("$MY_FILE")) end + )}, + {"dir no {}", + with_env_fn( + "MY_DIR", + "/mydir", + fun() -> ?assertEqual("/mydir/foobar", Do(<<"$MY_DIR/foobar">>)) end + )}, + {"dir with {}", + with_env_fn( + "MY_DIR", + "/mydir", + fun() -> ?assertEqual("/mydir/foobar", Do(<<"${MY_DIR}/foobar">>)) end + )}, + %% a trailing / should not cause the sub path to become absolute + {"env dir with trailing /", + with_env_fn( + "MY_DIR", + "/mydir//", + fun() -> ?assertEqual("/mydir/foobar", Do(<<"${MY_DIR}/foobar">>)) end + )}, + {"string dir with doulbe /", + with_env_fn( + "MY_DIR", + "/mydir/", + fun() -> ?assertEqual("/mydir/foobar", Do(<<"${MY_DIR}//foobar">>)) end + )}, + {"env not found", + with_env_fn( + "MY_DIR", + "/mydir/", + fun() -> ?assertEqual("${MY_DIR2}//foobar", Do(<<"${MY_DIR2}//foobar">>)) end + )} + ]. + +with_env_fn(Name, Value, F) -> + fun() -> + with_envs(F, [{Name, Value}]) + end. + +with_envs(Fun, Envs) -> + with_envs(Fun, [], Envs). + +with_envs(Fun, Args, [{_Name, _Value} | _] = Envs) -> + set_envs(Envs), + try + apply(Fun, Args) + after + unset_envs(Envs) + end. + +set_envs([{_Name, _Value} | _] = Envs) -> + lists:map(fun({Name, Value}) -> os:putenv(Name, Value) end, Envs). + +unset_envs([{_Name, _Value} | _] = Envs) -> + lists:map(fun({Name, _}) -> os:unsetenv(Name) end, Envs). From 5acf0e281eaf93fecbe80ad4ad32d5d0090ece31 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 14:21:54 +0200 Subject: [PATCH 136/194] refactor: delete default authz config from emqx.conf --- apps/emqx_authz/etc/emqx_authz.conf | 10 ---------- apps/emqx_authz/src/emqx_authz_file.erl | 3 ++- apps/emqx_authz/src/emqx_authz_schema.erl | 9 ++++++++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 3bdc180c5..167b12b3f 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -2,14 +2,4 @@ authorization { deny_action = ignore no_match = allow cache = { enable = true } - sources = [ - { - type = file - enable = true - # This file is immutable to EMQX. - # Once new rules are created from dashboard UI or HTTP API, - # the file 'data/authz/acl.conf' is used instead of this one - path = "{{ platform_etc_dir }}/acl.conf" - } - ] } diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index ede4a9582..63e7be781 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -38,7 +38,8 @@ description() -> "AuthZ with static rules". -create(#{path := Path} = Source) -> +create(#{path := Path0} = Source) -> + Path = emqx_schema:naive_env_interpolation(Path0), Rules = case file:consult(Path) of {ok, Terms} -> diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 39bbcc360..280b9b16c 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -491,7 +491,7 @@ authz_fields() -> ?HOCON( ?ARRAY(?UNION(UnionMemberSelector)), #{ - default => [], + default => [default_authz()], desc => ?DESC(sources), %% doc_lift is force a root level reference instead of nesting sub-structs extra => #{doc_lift => true}, @@ -501,3 +501,10 @@ authz_fields() -> } )} ]. + +default_authz() -> + #{ + <<"type">> => <<"file">>, + <<"enable">> => true, + <<"path">> => <<"${EMQX_ETC_DIR}/acl.conf">> + }. From e0dc5645ab408476350a7c86a2dc2e38094c2e4e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 16:01:14 +0200 Subject: [PATCH 137/194] fix(config): always fill defautls for all roots prior to this commit, if a root existed in config files it skips populating default values in raw config, this made impossible to add default values for authz sources. --- apps/emqx/src/emqx_config.erl | 67 ++++++++++++++++++----------------- apps/emqx/src/emqx_schema.erl | 1 + 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 9561263ca..c117bffba 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -35,7 +35,6 @@ save_to_config_map/2, save_to_override_conf/3 ]). --export([raw_conf_with_default/4]). -export([merge_envs/2]). -export([ @@ -329,7 +328,7 @@ init_load(SchemaMod, ConfFiles) -> -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> HasDeprecatedFile = has_deprecated_file(), - RawConf = parse_hocon(HasDeprecatedFile, Conf), + RawConf = load_config_files(HasDeprecatedFile, Conf), init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts). init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> @@ -339,18 +338,16 @@ init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> RawConfWithEnvs = merge_envs(SchemaMod, RawConf), Overrides = read_override_confs(), RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + RawConfAll = maybe_fill_defaults(SchemaMod, RawConfWithOverrides, Opts), %% check configs against the schema {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), save_to_app_env(AppEnvs), ok = save_to_config_map(CheckedConf, RawConfAll); init_load(false, SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - RootNames = get_root_names(), %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + RawConfAll = maybe_fill_defaults(SchemaMod, RawConfWithEnvs, Opts), %% check configs against the schema {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), save_to_app_env(AppEnvs), @@ -363,47 +360,53 @@ read_override_confs() -> hocon:deep_merge(ClusterOverrides, LocalOverrides). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. -raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> - Fun = fun(Name, Acc) -> - case maps:is_key(Name, RawConf) of - true -> - Acc; - false -> - case lists:keyfind(Name, 1, hocon_schema:roots(SchemaMod)) of - false -> - Acc; - {_, {_, Schema}} -> - Acc#{Name => schema_default(Schema)} - end - end - end, - RawDefault = lists:foldl(Fun, #{}, RootNames), - maps:merge(RawConf, fill_defaults(SchemaMod, RawDefault, #{})); -raw_conf_with_default(_SchemaMod, _RootNames, RawConf, _Opts) -> +maybe_fill_defaults(SchemaMod, RawConf0, #{raw_with_default := true}) -> + RootSchemas = hocon_schema:roots(SchemaMod), + %% the roots which are missing from the loaded configs + MissingRoots = lists:filtermap( + fun({BinName, Sc}) -> + case maps:is_key(BinName, RawConf0) of + true -> false; + false -> {true, Sc} + end + end, + RootSchemas + ), + RawConf = lists:foldl( + fun({RootName, Schema}, Acc) -> + Acc#{bin(RootName) => seed_default(Schema)} + end, + RawConf0, + MissingRoots + ), + fill_defaults(RawConf); +maybe_fill_defaults(_SchemaMod, RawConf, _Opts) -> RawConf. -schema_default(Schema) -> - case hocon_schema:field_schema(Schema, type) of - ?ARRAY(_) -> - []; - _ -> - #{} +%% if a root is not found in the raw conf, fill it with default values. +seed_default(Schema) -> + case hocon_schema:field_schema(Schema, default) of + undefined -> + %% so far all roots without a default value are objects + #{}; + Value -> + Value end. -parse_hocon(HasDeprecatedFile, Conf) -> +load_config_files(HasDeprecatedFile, Conf) -> IncDirs = include_dirs(), case do_parse_hocon(HasDeprecatedFile, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> ?SLOG(error, #{ - msg => "failed_to_load_hocon_file", + msg => "failed_to_load_config_file", reason => Reason, pwd => file:get_cwd(), include_dirs => IncDirs, config_file => Conf }), - error(failed_to_load_hocon_file) + error(failed_to_load_config_file) end. do_parse_hocon(true, Conf, IncDirs) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8fdd2557e..db3046bcf 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2794,6 +2794,7 @@ authentication(Which) -> hoconsc:mk(Type, #{ desc => Desc, converter => fun ensure_array/2, + default => [], importance => Importance }). From a1551213c892325b076b62a99085da7c05300e1b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 16:23:32 +0200 Subject: [PATCH 138/194] test: EMQX_ETC_DIR for test is app's etc dir --- apps/emqx/src/emqx_schema.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index db3046bcf..c1fe656da 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3181,5 +3181,13 @@ resolve_env(Name0) -> true -> {ok, Value}; false -> - error + special_env(Name) end. + +-ifdef(TEST). +%% when running tests, we need to mock the env variables +special_env("EMQX_ETC_DIR") -> + {ok, filename:join([code:lib_dir(emqx), etc])}. +-else. +special_env(_Name) -> error. +-endif. From 41f13330ba42825f8494c8cd7f896c8ee829616f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 16:29:25 +0200 Subject: [PATCH 139/194] refactor: export EMQX_LOG_DIR --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- bin/emqx | 10 +++++----- bin/node_dump | 10 +++++----- rel/emqx_vars | 3 ++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 39dce0b71..754b3c3fc 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1199,7 +1199,7 @@ log_handler_common_confs(Enable) -> ]. crash_dump_file_default() -> - case os:getenv("RUNNER_LOG_DIR") of + case os:getenv("EMQX_LOG_DIR") of false -> %% testing, or running emqx app as deps <<"log/erl_crash.dump">>; diff --git a/bin/emqx b/bin/emqx index fc0124b96..a181ae6a9 100755 --- a/bin/emqx +++ b/bin/emqx @@ -304,7 +304,7 @@ if [ "$ES" -ne 0 ]; then fi # Make sure log directory exists -mkdir -p "$RUNNER_LOG_DIR" +mkdir -p "$EMQX_LOG_DIR" # turn off debug as this is static set +x @@ -757,7 +757,7 @@ generate_config() { local node_name="$2" ## Delete the *.siz files first or it can't start after ## changing the config 'log.rotation.size' - rm -f "${RUNNER_LOG_DIR}"/*.siz + rm -f "${EMQX_LOG_DIR}"/*.siz ## timestamp for each generation local NOW_TIME @@ -979,7 +979,7 @@ diagnose_boot_failure_and_die() { local ps_line ps_line="$(find_emqx_process)" if [ -z "$ps_line" ]; then - echo "Find more information in the latest log file: ${RUNNER_LOG_DIR}/erlang.log.*" + echo "Find more information in the latest log file: ${EMQX_LOG_DIR}/erlang.log.*" exit 1 fi if ! relx_nodetool "ping" > /dev/null; then @@ -990,7 +990,7 @@ diagnose_boot_failure_and_die() { fi if ! relx_nodetool 'eval' 'true = emqx:is_running()' > /dev/null; then logerr "$NAME node is started, but failed to complete the boot sequence in time." - echo "Please collect the logs in ${RUNNER_LOG_DIR} and report a bug to EMQX team at https://github.com/emqx/emqx/issues/new/choose" + echo "Please collect the logs in ${EMQX_LOG_DIR} and report a bug to EMQX team at https://github.com/emqx/emqx/issues/new/choose" pipe_shutdown exit 3 fi @@ -1065,7 +1065,7 @@ case "${COMMAND}" in mkdir -p "$PIPE_DIR" - "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \ + "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$EMQX_LOG_DIR" \ "$(relx_start_command)" WAIT_TIME=${EMQX_WAIT_FOR_START:-120} diff --git a/bin/node_dump b/bin/node_dump index 1c4df08b5..60c995885 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -10,10 +10,10 @@ echo "Running node dump in ${RUNNER_ROOT_DIR}" cd "${RUNNER_ROOT_DIR}" -DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" -CONF_DUMP="$RUNNER_LOG_DIR/conf.dump" -LICENSE_INFO="$RUNNER_LOG_DIR/license_info.txt" -SYSINFO="$RUNNER_LOG_DIR/sysinfo.txt" +DUMP="$EMQX_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" +CONF_DUMP="$EMQX_LOG_DIR/conf.dump" +LICENSE_INFO="$EMQX_LOG_DIR/license_info.txt" +SYSINFO="$EMQX_LOG_DIR/sysinfo.txt" LOG_MAX_AGE_DAYS=3 @@ -74,7 +74,7 @@ done # Pack files { - find "$RUNNER_LOG_DIR" -mtime -"${LOG_MAX_AGE_DAYS}" \( -name '*.log.*' -or -name 'run_erl.log*' \) + find "$EMQX_LOG_DIR" -mtime -"${LOG_MAX_AGE_DAYS}" \( -name '*.log.*' -or -name 'run_erl.log*' \) echo "${SYSINFO}" echo "${CONF_DUMP}" echo "${LICENSE_INFO}" diff --git a/rel/emqx_vars b/rel/emqx_vars index e3965d40c..f37968f1f 100644 --- a/rel/emqx_vars +++ b/rel/emqx_vars @@ -11,7 +11,8 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" IS_ELIXIR="${IS_ELIXIR:-{{ is_elixir }}}" ## Allow users to pre-set `EMQX_LOG_DIR` because it only affects boot commands like `start` and `console`, ## but not other commands such as `ping` and `ctl`. -RUNNER_LOG_DIR="${EMQX_LOG_DIR:-${RUNNER_LOG_DIR:-{{ runner_log_dir }}}}" +## RUNNER_LOG_DIR is kept for backward compatibility. +export EMQX_LOG_DIR="${EMQX_LOG_DIR:-${RUNNER_LOG_DIR:-{{ runner_log_dir }}}}" EMQX_ETC_DIR="{{ emqx_etc_dir }}" RUNNER_USER="{{ runner_user }}" SCHEMA_MOD="{{ emqx_schema_mod }}" From 4d705817d85ce343dd8ec25fe5075090fba3cea8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 27 Apr 2023 16:35:39 +0200 Subject: [PATCH 140/194] refactor(log): move default values to schema --- apps/emqx/src/emqx_schema.erl | 7 ++- apps/emqx/test/emqx_common_test_helpers.erl | 3 +- apps/emqx/test/emqx_listeners_SUITE.erl | 3 +- apps/emqx_conf/etc/emqx_conf.conf | 7 --- apps/emqx_conf/src/emqx_conf_schema.erl | 63 +++++++++++++++++---- bin/emqx | 14 +++-- deploy/docker/docker-entrypoint.sh | 6 +- deploy/packages/emqx.service | 4 +- mix.exs | 2 - rebar.config.erl | 2 - rel/i18n/emqx_conf_schema.hocon | 9 ++- 11 files changed, 78 insertions(+), 42 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c1fe656da..418a2db56 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3187,7 +3187,12 @@ resolve_env(Name0) -> -ifdef(TEST). %% when running tests, we need to mock the env variables special_env("EMQX_ETC_DIR") -> - {ok, filename:join([code:lib_dir(emqx), etc])}. + {ok, filename:join([code:lib_dir(emqx), etc])}; +special_env("EMQX_LOG_DIR") -> + {ok, "log"}; +special_env(_Name) -> + %% only in tests + error. -else. special_env(_Name) -> error. -endif. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index e9ddc61a8..c6b04eed1 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -271,8 +271,7 @@ mustache_vars(App, Opts) -> ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}), Defaults = #{ platform_data_dir => app_path(App, "data"), - platform_etc_dir => app_path(App, "etc"), - platform_log_dir => app_path(App, "log") + platform_etc_dir => app_path(App, "etc") }, maps:merge(Defaults, ExtraMustacheVars). diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 107f3d4e7..8d965e8dd 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -266,8 +266,7 @@ render_config_file() -> mustache_vars() -> [ {platform_data_dir, local_path(["data"])}, - {platform_etc_dir, local_path(["etc"])}, - {platform_log_dir, local_path(["log"])} + {platform_etc_dir, local_path(["etc"])} ]. generate_config() -> diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 4c03c10c6..76e3c0805 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -15,13 +15,6 @@ node { data_dir = "{{ platform_data_dir }}" } -log { - file_handlers.default { - level = warning - file = "{{ platform_log_dir }}/emqx.log" - } -} - cluster { name = emqxcl discovery_strategy = manual diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 754b3c3fc..ee718b3a1 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -93,7 +93,10 @@ roots() -> {"log", sc( ?R_REF("log"), - #{translate_to => ["kernel"]} + #{ + translate_to => ["kernel"], + importance => ?IMPORTANCE_HIGH + } )}, {"rpc", sc( @@ -862,15 +865,25 @@ fields("rpc") -> ]; fields("log") -> [ - {"console_handler", ?R_REF("console_handler")}, + {"console_handler", + sc( + ?R_REF("console_handler"), + #{importance => ?IMPORTANCE_HIGH} + )}, {"file_handlers", sc( map(name, ?R_REF("log_file_handler")), - #{desc => ?DESC("log_file_handlers")} + #{ + desc => ?DESC("log_file_handlers"), + %% because file_handlers is a map + %% so there has to be a default value in order to populate the raw configs + default => #{<<"default">> => #{<<"level">> => <<"warning">>}}, + importance => ?IMPORTANCE_HIGH + } )} ]; fields("console_handler") -> - log_handler_common_confs(false); + log_handler_common_confs(console); fields("log_file_handler") -> [ {"file", @@ -878,6 +891,8 @@ fields("log_file_handler") -> file(), #{ desc => ?DESC("log_file_handler_file"), + default => <<"${EMQX_LOG_DIR}/emqx.log">>, + converter => fun emqx_schema:naive_env_interpolation/1, validator => fun validate_file_location/1 } )}, @@ -891,10 +906,11 @@ fields("log_file_handler") -> hoconsc:union([infinity, emqx_schema:bytesize()]), #{ default => <<"50MB">>, - desc => ?DESC("log_file_handler_max_size") + desc => ?DESC("log_file_handler_max_size"), + importance => ?IMPORTANCE_MEDIUM } )} - ] ++ log_handler_common_confs(true); + ] ++ log_handler_common_confs(file); fields("log_rotation") -> [ {"enable", @@ -1103,14 +1119,33 @@ tr_logger_level(Conf) -> tr_logger_handlers(Conf) -> emqx_config_logger:tr_handlers(Conf). -log_handler_common_confs(Enable) -> +log_handler_common_confs(Handler) -> + lists:map( + fun + ({_Name, #{importance := _}} = F) -> F; + ({Name, Sc}) -> {Name, Sc#{importance => ?IMPORTANCE_LOW}} + end, + do_log_handler_common_confs(Handler) + ). +do_log_handler_common_confs(Handler) -> + %% we rarely support dynamic defaults like this + %% for this one, we have build-time defualut the same as runtime default + %% so it's less tricky + EnableValues = + case Handler of + console -> ["console", "both"]; + file -> ["file", "both", "", false] + end, + EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"), + Enable = lists:member(EnvValue, EnableValues), [ {"enable", sc( boolean(), #{ default => Enable, - desc => ?DESC("common_handler_enable") + desc => ?DESC("common_handler_enable"), + importance => ?IMPORTANCE_LOW } )}, {"level", @@ -1127,7 +1162,8 @@ log_handler_common_confs(Enable) -> #{ default => <<"system">>, desc => ?DESC("common_handler_time_offset"), - validator => fun validate_time_offset/1 + validator => fun validate_time_offset/1, + importance => ?IMPORTANCE_LOW } )}, {"chars_limit", @@ -1135,7 +1171,8 @@ log_handler_common_confs(Enable) -> hoconsc:union([unlimited, range(100, inf)]), #{ default => unlimited, - desc => ?DESC("common_handler_chars_limit") + desc => ?DESC("common_handler_chars_limit"), + importance => ?IMPORTANCE_LOW } )}, {"formatter", @@ -1143,7 +1180,8 @@ log_handler_common_confs(Enable) -> hoconsc:enum([text, json]), #{ default => text, - desc => ?DESC("common_handler_formatter") + desc => ?DESC("common_handler_formatter"), + importance => ?IMPORTANCE_MEDIUM } )}, {"single_line", @@ -1151,7 +1189,8 @@ log_handler_common_confs(Enable) -> boolean(), #{ default => true, - desc => ?DESC("common_handler_single_line") + desc => ?DESC("common_handler_single_line"), + importance => ?IMPORTANCE_LOW } )}, {"sync_mode_qlen", diff --git a/bin/emqx b/bin/emqx index a181ae6a9..dd0b0791e 100755 --- a/bin/emqx +++ b/bin/emqx @@ -861,7 +861,13 @@ wait_until_return_val() { done } -# backward compatible with 4.x +# First, there is EMQX_DEFAULT_LOG_HANDLER which can control the default values +# to be used when generating configs. +# It's set in docker entrypoint and in systemd service file. +# +# To be backward compatible with 4.x and v5.0.0 ~ v5.0.24/e5.0.2: +# if EMQX_LOG__TO is set, we try to enable handlers from environment variables. +# i.e. it overrides the default value set in EMQX_DEFAULT_LOG_HANDLER tr_log_to_env() { local log_to=${EMQX_LOG__TO:-undefined} # unset because it's unknown to 5.0 @@ -893,13 +899,11 @@ tr_log_to_env() { maybe_log_to_console() { if [ "${EMQX_LOG__TO:-}" = 'default' ]; then - # want to use config file defaults, do nothing + # want to use defaults, do nothing unset EMQX_LOG__TO else tr_log_to_env - # ensure defaults - export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}" - export EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE="${EMQX_LOG__FILE_HANDLERS__DEFAULT__ENABLE:-false}" + export EMQX_DEFAULT_LOG_HANDLER=${EMQX_DEFAULT_LOG_HANDLER:-console} fi } diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 1824e1ee0..056f0675f 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash -## EMQ docker image start script -# Huang Rui -# EMQX Team -## Shell setting +## EMQ docker image start script + if [[ -n "$DEBUG" ]]; then set -ex else diff --git a/deploy/packages/emqx.service b/deploy/packages/emqx.service index d826e358b..2dbe550bc 100644 --- a/deploy/packages/emqx.service +++ b/deploy/packages/emqx.service @@ -10,8 +10,8 @@ Group=emqx Type=simple Environment=HOME=/var/lib/emqx -# Enable logging to file -Environment=EMQX_LOG__TO=default +# log to file by default (if no log handler config) +Environment=EMQX_DEFAULT_LOG_HANDLER=file # Start 'foreground' but not 'start' (daemon) mode. # Because systemd monitor/restarts 'simple' services diff --git a/mix.exs b/mix.exs index 93f7417c9..41a35e0e7 100644 --- a/mix.exs +++ b/mix.exs @@ -665,7 +665,6 @@ defmodule EMQXUmbrella.MixProject do emqx_default_erlang_cookie: default_cookie(), platform_data_dir: "data", platform_etc_dir: "etc", - platform_log_dir: "log", platform_plugins_dir: "plugins", runner_bin_dir: "$RUNNER_ROOT_DIR/bin", emqx_etc_dir: "$RUNNER_ROOT_DIR/etc", @@ -688,7 +687,6 @@ defmodule EMQXUmbrella.MixProject do emqx_default_erlang_cookie: default_cookie(), platform_data_dir: "/var/lib/emqx", platform_etc_dir: "/etc/emqx", - platform_log_dir: "/var/log/emqx", platform_plugins_dir: "/var/lib/emqx/plugins", runner_bin_dir: "/usr/bin", emqx_etc_dir: "/etc/emqx", diff --git a/rebar.config.erl b/rebar.config.erl index 7c00622c2..9ef6c5e00 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -335,7 +335,6 @@ overlay_vars_pkg(bin) -> [ {platform_data_dir, "data"}, {platform_etc_dir, "etc"}, - {platform_log_dir, "log"}, {platform_plugins_dir, "plugins"}, {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}, {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"}, @@ -348,7 +347,6 @@ overlay_vars_pkg(pkg) -> [ {platform_data_dir, "/var/lib/emqx"}, {platform_etc_dir, "/etc/emqx"}, - {platform_log_dir, "/var/log/emqx"}, {platform_plugins_dir, "/var/lib/emqx/plugins"}, {runner_bin_dir, "/usr/bin"}, {emqx_etc_dir, "/etc/emqx"}, diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index b252353f8..9cff400e6 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -1369,9 +1369,12 @@ this is where to look.""" desc_log { desc { - en: """EMQX logging supports multiple sinks for the log events. -Each sink is represented by a _log handler_, which can be configured independently.""" - zh: """EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。""" + en: """EMQX supports multiple log handlers, one console handler and multiple file handlers. +EMQX by default logs to console when running in docker or in console/foreground mode, +otherwise it logs to file $EMQX_LOG_DIR/emqx.log. +For advanced configuration, you can find more parameters in this section.""" + zh: """EMQX 支持同时多个日志输出，一个控制台输出，和多个文件输出。 +默认情况下，EMQX 运行在容器中，或者在 'console' 或 'foreground' 模式下运行时，会输出到 控制台，否则输出到文件。""" } label { en: "Log" From dbf9bae7dcbc04726791f72d00d5fa098c589413 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 09:09:57 +0200 Subject: [PATCH 141/194] test(statsd): fix raw config default value --- apps/emqx_statsd/test/emqx_statsd_SUITE.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index b5669e4b9..1f7c4688e 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -92,8 +92,10 @@ t_server_validator(_) -> ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?DEFAULT_CONF, #{ raw_with_default => true }), - undefined = emqx_conf:get_raw([statsd, server], undefined), - ?assertMatch("127.0.0.1:8125", emqx_conf:get([statsd, server])), + DefaultServer = default_server(), + ?assertEqual(DefaultServer, emqx_conf:get_raw([statsd, server])), + DefaultServerStr = binary_to_list(DefaultServer), + ?assertEqual(DefaultServerStr, emqx_conf:get([statsd, server])), %% recover ok = emqx_common_test_helpers:load_config(emqx_statsd_schema, ?BASE_CONF, #{ raw_with_default => true @@ -204,3 +206,7 @@ request(Method, Body) -> {ok, _Status, _} -> error end. + +default_server() -> + {server, Schema} = lists:keyfind(server, 1, emqx_statsd_schema:fields("statsd")), + hocon_schema:field_schema(Schema, default). From b3c0abf4946660c3b58517b062c251a2caf829c4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 12:45:16 +0200 Subject: [PATCH 142/194] test(emqx_management): fix listeners api test cases --- .../test/emqx_mgmt_api_listeners_SUITE.erl | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 62f689a84..e73f67f95 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -20,18 +20,27 @@ -include_lib("eunit/include/eunit.hrl"). --define(PORT, (20000 + ?LINE)). +-define(PORT(Base), (Base + ?LINE)). +-define(PORT, ?PORT(20000)). all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + %% we have to materialize the config file with default values for this test suite + %% because we want to test the deletion of non-existing listener + %% if there is no config file, the such deletion would result in a deletion + %% of the default listener. + Name = atom_to_list(?MODULE) ++ "-default-listeners", + TmpConfFullPath = inject_tmp_config_content(Name, default_listeners_hcon_text()), emqx_mgmt_api_test_util:init_suite([emqx_conf]), - Config. + [{injected_conf_file, TmpConfFullPath} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_conf:remove([listeners, tcp, new], #{override_to => cluster}), emqx_conf:remove([listeners, tcp, new1], #{override_to => local}), + {_, File} = lists:keyfind(injected_conf_file, 1, Config), + ok = file:delete(File), emqx_mgmt_api_test_util:end_suite([emqx_conf]). init_per_testcase(Case, Config) -> @@ -52,17 +61,12 @@ end_per_testcase(Case, Config) -> t_max_connection_default({init, Config}) -> emqx_mgmt_api_test_util:end_suite([emqx_conf]), - Etc = filename:join(["etc", "emqx.conf.all"]), - TmpConfName = atom_to_list(?FUNCTION_NAME) ++ ".conf", - Inc = filename:join(["etc", TmpConfName]), - ConfFile = emqx_common_test_helpers:app_path(emqx_conf, Etc), - IncFile = emqx_common_test_helpers:app_path(emqx_conf, Inc), Port = integer_to_binary(?PORT), Bin = <<"listeners.tcp.max_connection_test {bind = \"0.0.0.0:", Port/binary, "\"}">>, - ok = file:write_file(IncFile, Bin), - ok = file:write_file(ConfFile, ["include \"", TmpConfName, "\""], [append]), + TmpConfName = atom_to_list(?FUNCTION_NAME) ++ ".conf", + TmpConfFullPath = inject_tmp_config_content(TmpConfName, Bin), emqx_mgmt_api_test_util:init_suite([emqx_conf]), - [{tmp_config_file, IncFile} | Config]; + [{tmp_config_file, TmpConfFullPath} | Config]; t_max_connection_default({'end', Config}) -> ok = file:delete(proplists:get_value(tmp_config_file, Config)); t_max_connection_default(Config) when is_list(Config) -> @@ -123,7 +127,7 @@ t_tcp_crud_listeners_by_id(Config) when is_list(Config) -> MinListenerId = <<"tcp:min">>, BadId = <<"tcp:bad">>, Type = <<"tcp">>, - crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type). + crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, 31000). t_ssl_crud_listeners_by_id(Config) when is_list(Config) -> ListenerId = <<"ssl:default">>, @@ -131,7 +135,7 @@ t_ssl_crud_listeners_by_id(Config) when is_list(Config) -> MinListenerId = <<"ssl:min">>, BadId = <<"ssl:bad">>, Type = <<"ssl">>, - crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type). + crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, 32000). t_ws_crud_listeners_by_id(Config) when is_list(Config) -> ListenerId = <<"ws:default">>, @@ -139,7 +143,7 @@ t_ws_crud_listeners_by_id(Config) when is_list(Config) -> MinListenerId = <<"ws:min">>, BadId = <<"ws:bad">>, Type = <<"ws">>, - crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type). + crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, 33000). t_wss_crud_listeners_by_id(Config) when is_list(Config) -> ListenerId = <<"wss:default">>, @@ -147,7 +151,7 @@ t_wss_crud_listeners_by_id(Config) when is_list(Config) -> MinListenerId = <<"wss:min">>, BadId = <<"wss:bad">>, Type = <<"wss">>, - crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type). + crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, 34000). t_api_listeners_list_not_ready(Config) when is_list(Config) -> net_kernel:start(['listeners@127.0.0.1', longnames]), @@ -266,16 +270,18 @@ cluster(Specs) -> end} ]). -crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type) -> +crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, PortBase) -> OriginPath = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), NewPath = emqx_mgmt_api_test_util:api_path(["listeners", NewListenerId]), OriginListener = request(get, OriginPath, [], []), + ct:pal("raw conf: ~p~n", [emqx_config:get_raw([listeners])]), + ct:pal("OriginListener:~p", [OriginListener]), %% create with full options ?assertEqual({error, not_found}, is_running(NewListenerId)), ?assertMatch({error, {"HTTP/1.1", 404, _}}, request(get, NewPath, [], [])), - Port1 = integer_to_binary(?PORT), - Port2 = integer_to_binary(?PORT), + Port1 = integer_to_binary(?PORT(PortBase)), + Port2 = integer_to_binary(?PORT(PortBase)), NewConf = OriginListener#{ <<"id">> => NewListenerId, <<"bind">> => <<"0.0.0.0:", Port1/binary>> @@ -417,3 +423,21 @@ data_file(Name) -> cert_file(Name) -> data_file(filename:join(["certs", Name])). + +default_listeners_hcon_text() -> + Sc = #{roots => emqx_schema:fields("listeners")}, + Listeners = hocon_tconf:make_serializable(Sc, #{}, #{}), + Config = #{<<"listeners">> => Listeners}, + hocon_pp:do(Config, #{}). + +%% inject a 'include' at the end of emqx.conf.all +%% the 'include' can be kept after test, +%% as long as the file has been deleted it is a no-op +inject_tmp_config_content(TmpFile, Content) -> + Etc = filename:join(["etc", "emqx.conf.all"]), + Inc = filename:join(["etc", TmpFile]), + ConfFile = emqx_common_test_helpers:app_path(emqx_conf, Etc), + TmpFileFullPath = emqx_common_test_helpers:app_path(emqx_conf, Inc), + ok = file:write_file(TmpFileFullPath, Content), + ok = file:write_file(ConfFile, ["\ninclude \"", TmpFileFullPath, "\"\n"], [append]), + TmpFileFullPath. From c13a972bf01326421f83e9143f585f08c0109781 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 12:57:29 +0200 Subject: [PATCH 143/194] test(emqx_management): add test group for listener API --- .../test/emqx_mgmt_api_listeners_SUITE.erl | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index e73f67f95..cb4e370d3 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -24,10 +24,29 @@ -define(PORT, ?PORT(20000)). all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, with_defaults_in_file}, + {group, without_defaults_in_file} + ]. + +groups() -> + AllTests = emqx_common_test_helpers:all(?MODULE), + [ + {with_defaults_in_file, AllTests}, + {without_defaults_in_file, AllTests} + ]. init_per_suite(Config) -> - %% we have to materialize the config file with default values for this test suite + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(without_defaults_in_file, Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf]), + Config; +init_per_group(with_defaults_in_file, Config) -> + %% we have to materialize the config file with default values for this test group %% because we want to test the deletion of non-existing listener %% if there is no config file, the such deletion would result in a deletion %% of the default listener. @@ -36,11 +55,16 @@ init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_conf]), [{injected_conf_file, TmpConfFullPath} | Config]. -end_per_suite(Config) -> +end_per_group(Group, Config) -> emqx_conf:remove([listeners, tcp, new], #{override_to => cluster}), emqx_conf:remove([listeners, tcp, new1], #{override_to => local}), - {_, File} = lists:keyfind(injected_conf_file, 1, Config), - ok = file:delete(File), + case Group =:= with_defaults_in_file of + true -> + {_, File} = lists:keyfind(injected_conf_file, 1, Config), + ok = file:delete(File); + false -> + ok + end, emqx_mgmt_api_test_util:end_suite([emqx_conf]). init_per_testcase(Case, Config) -> From 2e9dca280c2aeb5096dfa42d5f8cf02d27cbe9d5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 28 Apr 2023 15:22:20 +0200 Subject: [PATCH 144/194] refactor(listener-schema): use a tombstone for deleted listeners --- apps/emqx/src/emqx_config_handler.erl | 16 ++- apps/emqx/src/emqx_listeners.erl | 71 ++++++++---- apps/emqx/src/emqx_schema.erl | 106 ++++++++++++------ apps/emqx_conf/src/emqx_conf.erl | 5 + .../src/emqx_dashboard_swagger.erl | 5 +- .../src/emqx_mgmt_api_listeners.erl | 12 +- .../test/emqx_mgmt_api_listeners_SUITE.erl | 24 ++-- 7 files changed, 158 insertions(+), 81 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index e664a7dd7..adbba032a 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -447,11 +447,17 @@ merge_to_override_config(RawConf, Opts) -> up_req({remove, _Opts}) -> '$remove'; up_req({{update, Req}, _Opts}) -> Req. -return_change_result(ConfKeyPath, {{update, _Req}, Opts}) -> - #{ - config => emqx_config:get(ConfKeyPath), - raw_config => return_rawconf(ConfKeyPath, Opts) - }; +return_change_result(ConfKeyPath, {{update, Req}, Opts}) -> + case Req =/= emqx_schema:tombstone() of + true -> + #{ + config => emqx_config:get(ConfKeyPath), + raw_config => return_rawconf(ConfKeyPath, Opts) + }; + false -> + %% like remove, nothing to return + #{} + end; return_change_result(_ConfKeyPath, {remove, _Opts}) -> #{}. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 18ddcaba2..e00c79b60 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -22,7 +22,9 @@ -include("emqx_mqtt.hrl"). -include("logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). - +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. %% APIs -export([ list_raw/0, @@ -33,7 +35,8 @@ is_running/1, current_conns/2, max_conns/2, - id_example/0 + id_example/0, + default_max_conn/0 ]). -export([ @@ -61,8 +64,12 @@ -export([certs_dir/2]). -endif. +-type listener_id() :: atom() | binary(). + -define(CONF_KEY_PATH, [listeners, '?', '?']). -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]). +-define(MARK_DEL, marked_for_deletion). +-define(MARK_DEL_BIN, <<"marked_for_deletion">>). -spec id_example() -> atom(). id_example() -> 'tcp:default'. @@ -105,19 +112,22 @@ do_list_raw() -> format_raw_listeners({Type0, Conf}) -> Type = binary_to_atom(Type0), - lists:map( - fun({LName, LConf0}) when is_map(LConf0) -> - Bind = parse_bind(LConf0), - Running = is_running(Type, listener_id(Type, LName), LConf0#{bind => Bind}), - LConf1 = maps:remove(<<"authentication">>, LConf0), - LConf3 = maps:put(<<"running">>, Running, LConf1), - CurrConn = - case Running of - true -> current_conns(Type, LName, Bind); - false -> 0 - end, - LConf4 = maps:put(<<"current_connections">>, CurrConn, LConf3), - {Type0, LName, LConf4} + lists:filtermap( + fun + ({LName, LConf0}) when is_map(LConf0) -> + Bind = parse_bind(LConf0), + Running = is_running(Type, listener_id(Type, LName), LConf0#{bind => Bind}), + LConf1 = maps:remove(<<"authentication">>, LConf0), + LConf3 = maps:put(<<"running">>, Running, LConf1), + CurrConn = + case Running of + true -> current_conns(Type, LName, Bind); + false -> 0 + end, + LConf4 = maps:put(<<"current_connections">>, CurrConn, LConf3), + {true, {Type0, LName, LConf4}}; + ({_LName, _MarkDel}) -> + false end, maps:to_list(Conf) ). @@ -195,7 +205,7 @@ start() -> ok = emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), foreach_listeners(fun start_listener/3). --spec start_listener(atom()) -> ok | {error, term()}. +-spec start_listener(listener_id()) -> ok | {error, term()}. start_listener(ListenerId) -> apply_on_listener(ListenerId, fun start_listener/3). @@ -246,7 +256,7 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) -> restart() -> foreach_listeners(fun restart_listener/3). --spec restart_listener(atom()) -> ok | {error, term()}. +-spec restart_listener(listener_id()) -> ok | {error, term()}. restart_listener(ListenerId) -> apply_on_listener(ListenerId, fun restart_listener/3). @@ -271,7 +281,7 @@ stop() -> _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), foreach_listeners(fun stop_listener/3). --spec stop_listener(atom()) -> ok | {error, term()}. +-spec stop_listener(listener_id()) -> ok | {error, term()}. stop_listener(ListenerId) -> apply_on_listener(ListenerId, fun stop_listener/3). @@ -419,7 +429,9 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> end. %% Update the listeners at runtime -pre_config_update([listeners, Type, Name], {create, NewConf}, undefined) -> +pre_config_update([listeners, Type, Name], {create, NewConf}, V) when + V =:= undefined orelse V =:= ?MARK_DEL_BIN +-> CertsDir = certs_dir(Type, Name), {ok, convert_certs(CertsDir, NewConf)}; pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) -> @@ -434,6 +446,8 @@ pre_config_update([listeners, Type, Name], {update, Request}, RawConf) -> pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) -> NewConf = emqx_utils_maps:deep_merge(RawConf, Updated), {ok, NewConf}; +pre_config_update([listeners, _Type, _Name], ?MARK_DEL, _RawConf) -> + {ok, ?MARK_DEL}; pre_config_update(_Path, _Request, RawConf) -> {ok, RawConf}. @@ -446,9 +460,9 @@ post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf #{enabled := true} -> restart_listener(Type, Name, {OldConf, NewConf}); _ -> ok end; -post_config_update([listeners, _Type, _Name], '$remove', undefined, undefined, _AppEnvs) -> - ok; -post_config_update([listeners, Type, Name], '$remove', undefined, OldConf, _AppEnvs) -> +post_config_update([listeners, Type, Name], Op, _, OldConf, _AppEnvs) when + Op =:= ?MARK_DEL andalso is_map(OldConf) +-> ok = unregister_ocsp_stapling_refresh(Type, Name), case stop_listener(Type, Name, OldConf) of ok -> @@ -611,6 +625,7 @@ format_bind(Bin) when is_binary(Bin) -> listener_id(Type, ListenerName) -> list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). +-spec parse_listener_id(listener_id()) -> {ok, #{type => atom(), name => atom()}} | {error, term()}. parse_listener_id(Id) -> case string:split(str(Id), ":", leading) of [Type, Name] -> @@ -836,3 +851,15 @@ unregister_ocsp_stapling_refresh(Type, Name) -> ListenerId = listener_id(Type, Name), emqx_ocsp_cache:unregister_listener(ListenerId), ok. + +%% There is currently an issue with frontend +%% infinity is not a good value for it, so we use 5m for now +default_max_conn() -> + %% TODO: <<"infinity">> + 5_000_000. + +-ifdef(TEST). +%% since it's a copy-paste. we need to ensure it's the same atom. +ensure_same_atom_test() -> + ?assertEqual(?MARK_DEL, emqx_schema:tombstone()). +-endif. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 418a2db56..e36da0e0a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -100,6 +100,13 @@ convert_servers/2 ]). +%% tombstone types +-export([ + tombstone/0, + tombstone_map/2, + get_tombstone_map_value_type/1 +]). + -behaviour(hocon_schema). -reflect_type([ @@ -777,45 +784,48 @@ fields("listeners") -> [ {"tcp", sc( - map(name, ref("mqtt_tcp_listener")), + tombstone_map(name, ref("mqtt_tcp_listener")), #{ desc => ?DESC(fields_listeners_tcp), - default => default_listener(tcp), + converter => fun(X, _) -> + ensure_default_listener(X, tcp) + end, required => {false, recursively} } )}, {"ssl", sc( - map(name, ref("mqtt_ssl_listener")), + tombstone_map(name, ref("mqtt_ssl_listener")), #{ desc => ?DESC(fields_listeners_ssl), - default => default_listener(ssl), + converter => fun(X, _) -> ensure_default_listener(X, ssl) end, required => {false, recursively} } )}, {"ws", sc( - map(name, ref("mqtt_ws_listener")), + tombstone_map(name, ref("mqtt_ws_listener")), #{ desc => ?DESC(fields_listeners_ws), - default => default_listener(ws), + converter => fun(X, _) -> ensure_default_listener(X, ws) end, required => {false, recursively} } )}, {"wss", sc( - map(name, ref("mqtt_wss_listener")), + tombstone_map(name, ref("mqtt_wss_listener")), #{ desc => ?DESC(fields_listeners_wss), - default => default_listener(wss), + converter => fun(X, _) -> ensure_default_listener(X, wss) end, required => {false, recursively} } )}, {"quic", sc( - map(name, ref("mqtt_quic_listener")), + tombstone_map(name, ref("mqtt_quic_listener")), #{ desc => ?DESC(fields_listeners_quic), + converter => fun keep_default_tombstone/2, required => {false, recursively} } )} @@ -1943,7 +1953,7 @@ base_listener(Bind) -> sc( hoconsc:union([infinity, pos_integer()]), #{ - default => <<"infinity">>, + default => emqx_listeners:default_max_conn(), desc => ?DESC(base_listener_max_connections) } )}, @@ -3092,20 +3102,12 @@ assert_required_field(Conf, Key, ErrorMessage) -> default_listener(tcp) -> #{ - <<"default">> => - #{ - <<"bind">> => <<"0.0.0.0:1883">>, - <<"max_connections">> => 1024000 - } + <<"bind">> => <<"0.0.0.0:1883">> }; default_listener(ws) -> #{ - <<"default">> => - #{ - <<"bind">> => <<"0.0.0.0:8083">>, - <<"max_connections">> => 1024000, - <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} - } + <<"bind">> => <<"0.0.0.0:8083">>, + <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} }; default_listener(SSLListener) -> %% The env variable is resolved in emqx_tls_lib by calling naive_env_interpolate @@ -3120,22 +3122,14 @@ default_listener(SSLListener) -> case SSLListener of ssl -> #{ - <<"default">> => - #{ - <<"bind">> => <<"0.0.0.0:8883">>, - <<"max_connections">> => 512000, - <<"ssl_options">> => SslOptions - } + <<"bind">> => <<"0.0.0.0:8883">>, + <<"ssl_options">> => SslOptions }; wss -> #{ - <<"default">> => - #{ - <<"bind">> => <<"0.0.0.0:8084">>, - <<"max_connections">> => 512000, - <<"ssl_options">> => SslOptions, - <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} - } + <<"bind">> => <<"0.0.0.0:8084">>, + <<"ssl_options">> => SslOptions, + <<"websocket">> => #{<<"mqtt_path">> => <<"/mqtt">>} } end. @@ -3196,3 +3190,47 @@ special_env(_Name) -> -else. special_env(_Name) -> error. -endif. + +%% The tombstone atom. +tombstone() -> + marked_for_deletion. + +%% Make a map type, the value of which is allowed to be 'marked_for_deletion' +%% 'marked_for_delition' is a special value which means the key is deleted. +%% This is used to support the 'delete' operation in configs, +%% since deleting the key would result in default value being used. +tombstone_map(Name, Type) -> + %% marked_for_deletion must be the last member of the union + %% because we need to first union member to populate the default values + map(Name, ?UNION([Type, tombstone()])). + +%% inverse of mark_del_map +get_tombstone_map_value_type(Schema) -> + %% TODO: violation of abstraction, expose an API in hoconsc + %% hoconsc:map_value_type(Schema) + ?MAP(_Name, Union) = hocon_schema:field_schema(Schema, type), + %% TODO: violation of abstraction, fix hoconsc:union_members/1 + ?UNION(Members) = Union, + Tombstone = tombstone(), + [Type, Tombstone] = hoconsc:union_members(Members), + Type. + +%% Keep the 'default' tombstone, but delete others. +keep_default_tombstone(Map, _Opts) when is_map(Map) -> + maps:filter( + fun(Key, Value) -> + Key =:= <<"default">> orelse Value =/= atom_to_binary(tombstone()) + end, + Map + ); +keep_default_tombstone(Value, _Opts) -> + Value. + +ensure_default_listener(undefined, ListenerType) -> + %% let the schema's default value do its job + #{<<"default">> => default_listener(ListenerType)}; +ensure_default_listener(#{<<"default">> := _} = Map, _ListenerType) -> + keep_default_tombstone(Map, #{}); +ensure_default_listener(Map, ListenerType) -> + NewMap = Map#{<<"default">> => default_listener(ListenerType)}, + keep_default_tombstone(NewMap, #{}). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 1ecda913d..1b37f652b 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -24,6 +24,7 @@ -export([get_by_node/2, get_by_node/3]). -export([update/3, update/4]). -export([remove/2, remove/3]). +-export([tombstone/2]). -export([reset/2, reset/3]). -export([dump_schema/1, dump_schema/3]). -export([schema_module/0]). @@ -107,6 +108,10 @@ update(Node, KeyPath, UpdateReq, Opts0) when Node =:= node() -> update(Node, KeyPath, UpdateReq, Opts) -> emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). +%% @doc Mark the specified key path as tombstone +tombstone(KeyPath, Opts) -> + update(KeyPath, emqx_schema:tombstone(), Opts). + %% @doc remove all value of key path in cluster-override.conf or local-override.conf. -spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index b2ad69997..627ef0719 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -237,8 +237,9 @@ parse_spec_ref(Module, Path, Options) -> erlang:apply(Module, schema, [Path]) %% better error message catch - error:Reason -> - throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}}) + error:Reason:Stacktrace -> + MoreInfo = #{module => Module, path => Path, reason => Reason}, + erlang:raise(error, MoreInfo, Stacktrace) end, {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index d7f3ff321..9a338b33f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -293,12 +293,14 @@ listeners_type() -> listeners_info(Opts) -> Listeners = hocon_schema:fields(emqx_schema, "listeners"), lists:map( - fun({Type, #{type := ?MAP(_Name, ?R_REF(Mod, Field))}}) -> - Fields0 = hocon_schema:fields(Mod, Field), + fun({ListenerType, Schema}) -> + Type = emqx_schema:get_tombstone_map_value_type(Schema), + ?R_REF(Mod, StructName) = Type, + Fields0 = hocon_schema:fields(Mod, StructName), Fields1 = lists:keydelete("authentication", 1, Fields0), Fields3 = required_bind(Fields1, Opts), - Ref = listeners_ref(Type, Opts), - TypeAtom = list_to_existing_atom(Type), + Ref = listeners_ref(ListenerType, Opts), + TypeAtom = list_to_existing_atom(ListenerType), #{ ref => ?R_REF(Ref), schema => [ @@ -642,7 +644,7 @@ create(Path, Conf) -> wrap(emqx_conf:update(Path, {create, Conf}, ?OPTS(cluster))). ensure_remove(Path) -> - wrap(emqx_conf:remove(Path, ?OPTS(cluster))). + wrap(emqx_conf:update(Path, emqx_schema:tombstone(), ?OPTS(cluster))). wrap({error, {post_config_update, emqx_listeners, Reason}}) -> {error, Reason}; wrap({error, {pre_config_update, emqx_listeners, Reason}}) -> {error, Reason}; diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index cb4e370d3..3a26c948e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -51,13 +51,13 @@ init_per_group(with_defaults_in_file, Config) -> %% if there is no config file, the such deletion would result in a deletion %% of the default listener. Name = atom_to_list(?MODULE) ++ "-default-listeners", - TmpConfFullPath = inject_tmp_config_content(Name, default_listeners_hcon_text()), + TmpConfFullPath = inject_tmp_config_content(Name, default_listeners_hocon_text()), emqx_mgmt_api_test_util:init_suite([emqx_conf]), [{injected_conf_file, TmpConfFullPath} | Config]. end_per_group(Group, Config) -> - emqx_conf:remove([listeners, tcp, new], #{override_to => cluster}), - emqx_conf:remove([listeners, tcp, new1], #{override_to => local}), + emqx_conf:tombstone([listeners, tcp, new], #{override_to => cluster}), + emqx_conf:tombstone([listeners, tcp, new1], #{override_to => local}), case Group =:= with_defaults_in_file of true -> {_, File} = lists:keyfind(injected_conf_file, 1, Config), @@ -94,16 +94,16 @@ t_max_connection_default({init, Config}) -> t_max_connection_default({'end', Config}) -> ok = file:delete(proplists:get_value(tmp_config_file, Config)); t_max_connection_default(Config) when is_list(Config) -> - %% Check infinity is binary not atom. #{<<"listeners">> := Listeners} = emqx_mgmt_api_listeners:do_list_listeners(), Target = lists:filter( fun(#{<<"id">> := Id}) -> Id =:= 'tcp:max_connection_test' end, Listeners ), - ?assertMatch([#{<<"max_connections">> := <<"infinity">>}], Target), + DefaultMaxConn = emqx_listeners:default_max_conn(), + ?assertMatch([#{<<"max_connections">> := DefaultMaxConn}], Target), NewPath = emqx_mgmt_api_test_util:api_path(["listeners", "tcp:max_connection_test"]), - ?assertMatch(#{<<"max_connections">> := <<"infinity">>}, request(get, NewPath, [], [])), - emqx_conf:remove([listeners, tcp, max_connection_test], #{override_to => cluster}), + ?assertMatch(#{<<"max_connections">> := DefaultMaxConn}, request(get, NewPath, [], [])), + emqx_conf:tombstone([listeners, tcp, max_connection_test], #{override_to => cluster}), ok. t_list_listeners(Config) when is_list(Config) -> @@ -114,7 +114,7 @@ t_list_listeners(Config) when is_list(Config) -> %% POST /listeners ListenerId = <<"tcp:default">>, - NewListenerId = <<"tcp:new">>, + NewListenerId = <<"tcp:new11">>, OriginPath = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), NewPath = emqx_mgmt_api_test_util:api_path(["listeners", NewListenerId]), @@ -128,7 +128,7 @@ t_list_listeners(Config) when is_list(Config) -> OriginListener2 = maps:remove(<<"id">>, OriginListener), Port = integer_to_binary(?PORT), NewConf = OriginListener2#{ - <<"name">> => <<"new">>, + <<"name">> => <<"new11">>, <<"bind">> => <<"0.0.0.0:", Port/binary>>, <<"max_connections">> := <<"infinity">> }, @@ -298,8 +298,6 @@ crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, Port OriginPath = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]), NewPath = emqx_mgmt_api_test_util:api_path(["listeners", NewListenerId]), OriginListener = request(get, OriginPath, [], []), - ct:pal("raw conf: ~p~n", [emqx_config:get_raw([listeners])]), - ct:pal("OriginListener:~p", [OriginListener]), %% create with full options ?assertEqual({error, not_found}, is_running(NewListenerId)), @@ -314,7 +312,7 @@ crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, Port ?assertEqual(lists:sort(maps:keys(OriginListener)), lists:sort(maps:keys(Create))), Get1 = request(get, NewPath, [], []), ?assertMatch(Create, Get1), - ?assert(is_running(NewListenerId)), + ?assertEqual({true, NewListenerId}, {is_running(NewListenerId), NewListenerId}), %% create with required options MinPath = emqx_mgmt_api_test_util:api_path(["listeners", MinListenerId]), @@ -448,7 +446,7 @@ data_file(Name) -> cert_file(Name) -> data_file(filename:join(["certs", Name])). -default_listeners_hcon_text() -> +default_listeners_hocon_text() -> Sc = #{roots => emqx_schema:fields("listeners")}, Listeners = hocon_tconf:make_serializable(Sc, #{}, #{}), Config = #{<<"listeners">> => Listeners}, From 03ae61569f8f2b3dfa59ee91b15575e353b9d581 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 30 Apr 2023 10:18:18 +0200 Subject: [PATCH 145/194] test(authn): fix test case after authentication default value added --- apps/emqx_authn/test/emqx_authn_schema_SUITE.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl index 7e67584ac..cd1c38d06 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl @@ -78,7 +78,8 @@ t_check_schema(_Config) -> ). t_union_member_selector(_) -> - ?assertMatch(#{authentication := undefined}, check(undefined)), + %% default value for authentication + ?assertMatch(#{authentication := []}, check(undefined)), C1 = #{<<"backend">> => <<"built_in_database">>}, ?assertThrow( #{ From 57cc854a4a0f7ff7e0f55e4c8dbe19ec370732c6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 30 Apr 2023 10:45:11 +0200 Subject: [PATCH 146/194] test(bridge): fix bridge map type filed converters now the converters on map type fields only work at the wrapping map level but not the values --- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 7 ++++++- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 12 +++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 4b9b7e3fe..f58805b6b 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -230,7 +230,12 @@ webhook_bridge_converter(Conf0, _HoconOpts) -> undefined -> undefined; _ -> - do_convert_webhook_config(Conf1) + maps:map( + fun(_Name, Conf) -> + do_convert_webhook_config(Conf) + end, + Conf1 + ) end. do_convert_webhook_config( diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 3ad5cbbb4..5981904c2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -181,7 +181,7 @@ kafka_structs() -> #{ desc => <<"Kafka Producer Bridge Config">>, required => false, - converter => fun emqx_bridge_kafka:kafka_producer_converter/2 + converter => fun kafka_producer_converter/2 } )}, {kafka_consumer, @@ -264,3 +264,13 @@ sqlserver_structs() -> } )} ]. + +kafka_producer_converter(undefined, _) -> + undefined; +kafka_producer_converter(Map, Opts) -> + maps:map( + fun(_Name, Config) -> + emqx_bridge_kafka:kafka_producer_converter(Config, Opts) + end, + Map + ). From 475dee32eea3196943617d9f445b2f1aa84d4581 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 30 Apr 2023 17:28:41 +0200 Subject: [PATCH 147/194] test(emqx_dashboard): refine spec error --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 14 ++++++++++---- .../test/emqx_swagger_requestBody_SUITE.erl | 4 ++-- .../test/emqx_swagger_response_SUITE.erl | 7 ++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 627ef0719..bffedaf3c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -235,11 +235,17 @@ parse_spec_ref(Module, Path, Options) -> Schema = try erlang:apply(Module, schema, [Path]) - %% better error message catch - error:Reason:Stacktrace -> - MoreInfo = #{module => Module, path => Path, reason => Reason}, - erlang:raise(error, MoreInfo, Stacktrace) + Error:Reason:Stacktrace -> + %% This error is intended to fail the build + %% hence print to standard_error + io:format( + standard_error, + "Failed to generate swagger for path ~p in module ~p~n" + "error:~p~nreason:~p~n~p~n", + [Module, Path, Error, Reason, Stacktrace] + ), + error({failed_to_generate_swagger_spec, Module, Path}) end, {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 3150ed097..4dbc8abdc 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -308,8 +308,8 @@ t_nest_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{mfa := {?MODULE, schema, [Path]}}}, + ?assertError( + {failed_to_generate_swagger_spec, ?MODULE, Path}, emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}) ), ok. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 4d1501dae..f357bdc8f 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -278,11 +278,8 @@ t_bad_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{ - mfa := {?MODULE, schema, ["/ref/none"]}, - reason := function_clause - }}, + ?assertError( + {failed_to_generate_swagger_spec, ?MODULE, Path}, validate(Path, #{}, []) ), ok. From 43c80ba635aa5a7922c3ae695a1d6a40ed7f37b6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 30 Apr 2023 21:24:46 +0200 Subject: [PATCH 148/194] chore: always init_load config wiht defaults populated this effectively eliminates the need for raw_with_default because it's now always set to true everywhere. will remove it in a followup. --- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- apps/emqx/test/emqx_logger_SUITE.erl | 1 - apps/emqx_conf/src/emqx_conf_app.erl | 7 ------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index c6b04eed1..95427942d 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -489,7 +489,7 @@ load_config(SchemaModule, Config, Opts) -> ok. load_config(SchemaModule, Config) -> - load_config(SchemaModule, Config, #{raw_with_default => false}). + load_config(SchemaModule, Config, #{raw_with_default => true}). -spec is_all_tcp_servers_available(Servers) -> Result when Servers :: [{Host, Port}], diff --git a/apps/emqx/test/emqx_logger_SUITE.erl b/apps/emqx/test/emqx_logger_SUITE.erl index c8ff63c75..e8d7d7a34 100644 --- a/apps/emqx/test/emqx_logger_SUITE.erl +++ b/apps/emqx/test/emqx_logger_SUITE.erl @@ -22,7 +22,6 @@ -include_lib("eunit/include/eunit.hrl"). -define(LOGGER, emqx_logger). --define(a, "a"). -define(SUPPORTED_LEVELS, [emergency, alert, critical, error, warning, notice, info, debug]). all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index dedb9aeab..2231b8336 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -88,15 +88,8 @@ sync_data_from_node() -> %% Internal functions %% ------------------------------------------------------------------------------ --ifdef(TEST). -init_load() -> - emqx_config:init_load(emqx_conf:schema_module(), #{raw_with_default => false}). - --else. - init_load() -> emqx_config:init_load(emqx_conf:schema_module(), #{raw_with_default => true}). --endif. init_conf() -> %% Workaround for https://github.com/emqx/mria/issues/94: From a03f2dd64bd2f9115d3ef0e59d35e120d7d8bae6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 1 May 2023 15:35:33 +0200 Subject: [PATCH 149/194] test: allow pre-load configs before emqx_conf app --- apps/emqx/src/emqx_config.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index c117bffba..3d5a9d2a4 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -360,12 +360,13 @@ read_override_confs() -> hocon:deep_merge(ClusterOverrides, LocalOverrides). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. +%% TODO: remove raw_with_default as it's now always true. maybe_fill_defaults(SchemaMod, RawConf0, #{raw_with_default := true}) -> RootSchemas = hocon_schema:roots(SchemaMod), %% the roots which are missing from the loaded configs MissingRoots = lists:filtermap( fun({BinName, Sc}) -> - case maps:is_key(BinName, RawConf0) of + case maps:is_key(BinName, RawConf0) orelse is_already_loaded(BinName) of true -> false; false -> {true, Sc} end @@ -383,6 +384,13 @@ maybe_fill_defaults(SchemaMod, RawConf0, #{raw_with_default := true}) -> maybe_fill_defaults(_SchemaMod, RawConf, _Opts) -> RawConf. +%% So far, this can only return true when testing. +%% e.g. when testing an app, we need to load its config first +%% then start emqx_conf application which will load the +%% possibly empty config again (then filled with defaults). +is_already_loaded(Name) -> + ?MODULE:get_raw([Name], #{}) =/= #{}. + %% if a root is not found in the raw conf, fill it with default values. seed_default(Schema) -> case hocon_schema:field_schema(Schema, default) of From 95cb262067db86f767b4ba08ded3695171f062d5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 1 May 2023 18:42:45 +0200 Subject: [PATCH 150/194] test: fix authn test cases --- apps/emqx/src/emqx_config.erl | 6 ++++-- apps/emqx/test/emqx_common_test_helpers.erl | 11 ++++++++++- apps/emqx/test/emqx_config_SUITE.erl | 4 ++-- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 2 +- apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl | 11 ++++++----- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 2 +- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 3d5a9d2a4..54648fca6 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -89,7 +89,7 @@ ]). -ifdef(TEST). --export([erase_schema_mod_and_names/0]). +-export([erase_all/0]). -endif. -include("logger.hrl"). @@ -559,7 +559,9 @@ save_schema_mod_and_names(SchemaMod) -> }). -ifdef(TEST). -erase_schema_mod_and_names() -> +erase_all() -> + Names = get_root_names(), + lists:foreach(fun erase/1, Names), persistent_term:erase(?PERSIS_SCHEMA_MODS). -endif. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 95427942d..7698a3389 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -249,11 +249,20 @@ start_app(App, SpecAppConfig, Opts) -> case application:ensure_all_started(App) of {ok, _} -> ok = ensure_dashboard_listeners_started(App), + ok = wait_for_app_processes(App), ok; {error, Reason} -> error({failed_to_start_app, App, Reason}) end. +wait_for_app_processes(emqx_conf) -> + %% emqx_conf app has a gen_server which + %% initializes its state asynchronously + gen_server:call(emqx_cluster_rpc, dummy), + ok; +wait_for_app_processes(_) -> + ok. + app_conf_file(emqx_conf) -> "emqx.conf.all"; app_conf_file(App) -> atom_to_list(App) ++ ".conf". @@ -309,7 +318,7 @@ stop_apps(Apps) -> %% to avoid inter-suite flakiness application:unset_env(emqx, init_config_load_done), persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), - emqx_config:erase_schema_mod_and_names(), + emqx_config:erase_all(), ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index 1704d1476..959c07dda 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -64,14 +64,14 @@ t_init_load(_Config) -> ConfFile = "./test_emqx.conf", ok = file:write_file(ConfFile, <<"">>), ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)), - emqx_config:erase_schema_mod_and_names(), + emqx_config:erase_all(), {ok, DeprecatedFile} = application:get_env(emqx, cluster_override_conf_file), ?assertEqual(false, filelib:is_regular(DeprecatedFile), DeprecatedFile), %% Don't has deprecated file ok = emqx_config:init_load(emqx_schema, [ConfFile]), ?assertEqual(ExpectRootNames, lists:sort(emqx_config:get_root_names())), ?assertMatch({ok, #{raw_config := 256}}, emqx:update_config([mqtt, max_topic_levels], 256)), - emqx_config:erase_schema_mod_and_names(), + emqx_config:erase_all(), %% Has deprecated file ok = file:write_file(DeprecatedFile, <<"{}">>), ok = emqx_config:init_load(emqx_schema, [ConfFile]), diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index c7f718dfc..6d9203c95 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -67,7 +67,7 @@ init_per_suite(Config) -> emqx_config:erase(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY), _ = application:load(emqx_conf), ok = emqx_mgmt_api_test_util:init_suite( - [emqx_authn] + [emqx_conf, emqx_authn] ), ?AUTHN:delete_chain(?GLOBAL), diff --git a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl index 59865ab41..98215e853 100644 --- a/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_enable_flag_SUITE.erl @@ -42,15 +42,16 @@ init_per_testcase(_Case, Config) -> <<"backend">> => <<"built_in_database">>, <<"user_id_type">> => <<"clientid">> }, - emqx:update_config( + {ok, _} = emqx:update_config( ?PATH, {create_authenticator, ?GLOBAL, AuthnConfig} ), - - emqx_conf:update( - [listeners, tcp, listener_authn_enabled], {create, listener_mqtt_tcp_conf(18830, true)}, #{} + {ok, _} = emqx_conf:update( + [listeners, tcp, listener_authn_enabled], + {create, listener_mqtt_tcp_conf(18830, true)}, + #{} ), - emqx_conf:update( + {ok, _} = emqx_conf:update( [listeners, tcp, listener_authn_disabled], {create, listener_mqtt_tcp_conf(18831, false)}, #{} diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 94c07ca96..bd18367b6 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -37,7 +37,7 @@ init_per_testcase(_, Config) -> init_per_suite(Config) -> _ = application:load(emqx_conf), - emqx_common_test_helpers:start_apps([emqx_authn]), + emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn]), application:ensure_all_started(emqx_resource), application:ensure_all_started(emqx_connector), Config. From b65a71b498ea219857c36d8765cea7fd7c88ee0b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 1 May 2023 19:35:57 +0200 Subject: [PATCH 151/194] test: allow emqx_ws_connection_SUITE to run without erasing configs --- apps/emqx/test/emqx_common_test_helpers.erl | 12 +++++++++++- apps/emqx/test/emqx_ws_connection_SUITE.erl | 8 ++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 7698a3389..26d908c5e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -31,6 +31,7 @@ start_apps/2, start_apps/3, stop_apps/1, + stop_apps/2, reload/2, app_path/2, proj_root/0, @@ -313,12 +314,21 @@ generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) -> -spec stop_apps(list()) -> ok. stop_apps(Apps) -> + stop_apps(Apps, #{}). + +stop_apps(Apps, Opts) -> [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]], ok = mria_mnesia:delete_schema(), %% to avoid inter-suite flakiness application:unset_env(emqx, init_config_load_done), persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY), - emqx_config:erase_all(), + case Opts of + #{erase_all_configs := false} -> + %% FIXME: this means inter-suite or inter-test dependencies + ok; + _ -> + emqx_config:erase_all() + end, ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index de8b1c9af..e62844547 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -138,13 +138,13 @@ end_per_testcase(t_ws_non_check_origin, Config) -> del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), - emqx_common_test_helpers:stop_apps([]), + stop_apps(), ok; end_per_testcase(_, Config) -> del_bucket(), PrevConfig = ?config(prev_config, Config), emqx_config:put_listener_conf(ws, default, [websocket], PrevConfig), - emqx_common_test_helpers:stop_apps([]), + stop_apps(), Config. init_per_suite(Config) -> @@ -156,6 +156,10 @@ end_per_suite(_) -> emqx_common_test_helpers:stop_apps([]), ok. +%% FIXME: this is a temp fix to tests share configs. +stop_apps() -> + emqx_common_test_helpers:stop_apps([], #{erase_all_configs => false}). + %%-------------------------------------------------------------------- %% Test Cases %%-------------------------------------------------------------------- From a3b1664c06a38ae4c8925bf689c93e283f9e6752 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 1 May 2023 20:15:47 +0200 Subject: [PATCH 152/194] test: allow inter-test case config dependency for emqx_exhook_SUITE --- apps/emqx_exhook/test/emqx_exhook_SUITE.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index d12f99917..ff313c8c8 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -301,10 +301,10 @@ t_cluster_name(_) -> ok end, - emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]), + stop_apps([emqx, emqx_exhook]), emqx_common_test_helpers:start_apps([emqx, emqx_exhook], SetEnvFun), on_exit(fun() -> - emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]), + stop_apps([emqx, emqx_exhook]), load_cfg(?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_exhook]), mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]) @@ -489,3 +489,7 @@ data_file(Name) -> cert_file(Name) -> data_file(filename:join(["certs", Name])). + +%% FIXME: this creats inter-test dependency +stop_apps(Apps) -> + emqx_common_test_helpers:stop_apps(Apps, #{erase_all_configs => false}). From 516c52bdc783a7f044846adb0c920d7bc262425c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 30 Apr 2023 10:48:20 +0200 Subject: [PATCH 153/194] build: add a 'app' build target which only compiles the code but no release --- Makefile | 5 +++++ build | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a39f44c07..e60e8e492 100644 --- a/Makefile +++ b/Makefile @@ -140,6 +140,11 @@ COMMON_DEPS := $(REBAR) $(REL_PROFILES:%=%): $(COMMON_DEPS) @$(BUILD) $(@) rel +.PHONY: compile $(PROFILES:%=compile-%) +compile: $(PROFILES:%=compile-%) +$(PROFILES:%=compile-%): + @$(BUILD) $(@:compile-%=%) apps + ## Not calling rebar3 clean because ## 1. rebar3 clean relies on rebar3, meaning it reads config, fetches dependencies etc. ## 2. it's slow diff --git a/build b/build index eb75317cc..2924f8a6f 100755 --- a/build +++ b/build @@ -124,16 +124,19 @@ assert_no_compile_time_only_deps() { : } -make_rel() { - local release_or_tar="${1}" +just_compile() { ./scripts/pre-compile.sh "$PROFILE" # make_elixir_rel always create rebar.lock # delete it to make git clone + checkout work because we use shallow close for rebar deps rm -f rebar.lock # compile all beams ./rebar3 as "$PROFILE" compile - # generate docs (require beam compiled), generated to etc and priv dirs make_docs +} + +make_rel() { + local release_or_tar="${1}" + just_compile # now assemble the release tar ./rebar3 as "$PROFILE" "$release_or_tar" assert_no_compile_time_only_deps @@ -375,6 +378,9 @@ export_elixir_release_vars() { log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in + apps) + just_compile + ;; doc|docs) make_docs ;; From dbcb75f35bae1d4a00122b3244f2123cdb1051fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 1 May 2023 21:34:33 +0200 Subject: [PATCH 154/194] build: add ./dev --- dev | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100755 dev diff --git a/dev b/dev new file mode 100755 index 000000000..69706bf6d --- /dev/null +++ b/dev @@ -0,0 +1,250 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { +cat <&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +case "${PROFILE}" in + ce|emqx) + PROFILE='emqx' + ;; + ee|emqx-enterprise) + PROFILE='emqx-enterprise' + ;; + *) + echo "Unknown profile $PROFILE" + exit 1 + ;; +esac +export PROFILE + +case "${PROFILE}" in + emqx) + SCHEMA_MOD='emqx_conf_schema' + ;; + emqx-enterprise) + SCHEMA_MOD='emqx_ee_conf_schema' + ;; +esac + +PROJ_ROOT="$(git rev-parse --show-toplevel)" +cd "$PROJ_ROOT" +BASE_DIR="_build/dev-run/$PROFILE" +export EMQX_ETC_DIR="$BASE_DIR/etc" +export EMQX_DATA_DIR="$BASE_DIR/data" +export EMQX_LOG_DIR="$BASE_DIR/log" +CONFIGS_DIR="$EMQX_DATA_DIR/configs" +COOKIE='emqxsecretcookie' +mkdir -p "$EMQX_ETC_DIR" "$EMQX_DATA_DIR/patches" "$EMQX_LOG_DIR" "$CONFIGS_DIR" + +## build compile the profile is it's not compiled yet +prepare_erl_libs() { + local profile="$1" + local libs_dir="_build/${profile}/lib" + local erl_libs='' + if [ $FORCE_COMPILE -eq 1 ] || [ ! -d "$libs_dir" ]; then + make "compile-${PROFILE}" + else + echo "Running from code in $libs_dir" + fi + for app in "${libs_dir}"/*; do + erl_libs="${erl_libs}:${app}" + done + export ERL_LIBS="$erl_libs" +} + +## poorman's mustache templating +mustache() { + local name="$1" + local value="$2" + local file="$3" + sed -i "s|{{\s*${name}\s*}}|${value}|g" "$file" +} + +## render the merged boot conf file. +## the merge action is done before the profile is compiled +render_hocon_conf() { + input="apps/emqx_conf/etc/emqx.conf.all" + output="$EMQX_ETC_DIR/emqx.conf" + cp "$input" "$output" + mustache emqx_default_erlang_cookie "$COOKIE" "$output" + mustache platform_data_dir "${EMQX_DATA_DIR}" "$output" + mustache platform_log_dir "${EMQX_LOG_DIR}" "$output" + mustache platform_etc_dir "${EMQX_ETC_DIR}" "$output" +} + +call_hocon() { + local in=("$@") + local args='' + for arg in "${in[@]}"; do + if [ -z "$args" ]; then + args="\"$arg\"" + else + args="$args, \"$arg\"" + fi + done + erl -noshell -eval "{ok, _} = application:ensure_all_started(hocon), ok = hocon_cli:main([$args]), init:stop()." +} + +# Function to generate app.config and vm.args +# sets two environment variables CONF_FILE and ARGS_FILE +generate_app_conf() { + ## timestamp for each generation + local NOW_TIME + NOW_TIME="$(date +'%Y.%m.%d.%H.%M.%S')" + + ## this command populates two files: app.