From ee25d9bd9e5c5c5db645de7b421d1c7ef0a28ce8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 30 Nov 2023 12:01:23 +0800 Subject: [PATCH 01/46] fix: vm.args use `multi_time_warp` See also: https://www.erlang.org/doc/apps/erts/time_correction#multi-time-warp-mode --- apps/emqx/etc/vm.args.cloud | 2 +- changes/fix-12059.en.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/fix-12059.en.md diff --git a/apps/emqx/etc/vm.args.cloud b/apps/emqx/etc/vm.args.cloud index 34fc19777..e8a1aef51 100644 --- a/apps/emqx/etc/vm.args.cloud +++ b/apps/emqx/etc/vm.args.cloud @@ -98,7 +98,7 @@ #+W w ## Sets time warp mode: no_time_warp | single_time_warp | multi_time_warp -#+C no_time_warp ++C multi_time_warp ## Prevents loading information about source filenames and line numbers. #+L diff --git a/changes/fix-12059.en.md b/changes/fix-12059.en.md new file mode 100644 index 000000000..288968309 --- /dev/null +++ b/changes/fix-12059.en.md @@ -0,0 +1,2 @@ +Use `multi-time-warp` as default time warp mode. +See also: [time_correction_#multi-time-warp-mode](https://www.erlang.org/doc/apps/erts/time_correction#multi-time-warp-mode) From 899fc7b9b94b5921c3c90f378beb8ce98960f00c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 1 Dec 2023 16:41:37 +0100 Subject: [PATCH 02/46] chore: bump ssl_verify_fun version to 1.1.7 --- mix.exs | 2 +- rebar.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index cd0424155..747f5a205 100644 --- a/mix.exs +++ b/mix.exs @@ -96,7 +96,7 @@ defmodule EMQXUmbrella.MixProject do {:gpb, "4.19.9", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, # set by hackney (dependency) - {:ssl_verify_fun, "1.1.6", override: true}, + {:ssl_verify_fun, "1.1.7", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, {:opentelemetry_api, diff --git a/rebar.config b/rebar.config index 5b05f25e9..e2cdf6018 100644 --- a/rebar.config +++ b/rebar.config @@ -92,6 +92,7 @@ , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_api_experimental"}} %% export , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_exporter"}} + , {ssl_verify_fun, "1.1.7"} ]}. {xref_ignores, From 9c30a7bbe753fc0880b23c156fc41cc3dd5cd050 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 1 Dec 2023 16:41:56 +0100 Subject: [PATCH 03/46] chore: fail early in pre-compile.sh when translation file is not available --- scripts/pre-compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 71b42d003..12506162c 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -32,7 +32,7 @@ DOWNLOAD_I18N_TRANSLATIONS=${DOWNLOAD_I18N_TRANSLATIONS:-true} if [ "$DOWNLOAD_I18N_TRANSLATIONS" = "true" ]; then echo "downloading i18n translation from emqx/emqx-i18n" start=$(date +%s) - curl -L --silent --show-error \ + curl -L --fail --silent --show-error \ --output "apps/emqx_dashboard/priv/desc.zh.hocon" \ "https://raw.githubusercontent.com/emqx/emqx-i18n/${I18N_REPO_BRANCH}/desc.zh.hocon" end=$(date +%s) From 0e27d031a8c730bbd5798b62c04022f59a5b992e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 1 Dec 2023 15:50:56 +0100 Subject: [PATCH 04/46] chore: 5.4.0-alpha.1 --- apps/emqx/include/emqx_release.hrl | 4 ++-- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- deploy/charts/emqx/Chart.yaml | 4 ++-- scripts/rel/cut.sh | 9 ++++++++- scripts/rel/sync-remotes.sh | 10 +++++++--- 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 299486ad1..83ed8ae56 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.3.2"). +-define(EMQX_RELEASE_CE, "5.4.0-alpha.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.3.2"). +-define(EMQX_RELEASE_EE, "5.4.0-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 652b2bcf5..5231d1c27 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.3.2 +version: 5.4.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.2 +appVersion: 5.4.0-alpha.1 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 9444fe14c..0d87c3251 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.3.2 +version: 5.4.0-alpha.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.3.2 +appVersion: 5.4.0-alpha.1 diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 5b0e855d1..e218f8d0b 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -23,6 +23,7 @@ options: release-51 release-52 release-53 + release-54 NOTE: this option should be used when --dryrun. --dryrun: Do not actually create the git tag. @@ -37,7 +38,7 @@ options: For 5.X series the current working branch must be 'release-5X' --.--[ master ]---------------------------.-----------.--- \\ / - \`---[release-53]----(v5.3.1 | e5.3.1) + \`---[release-54]----(v5.4.0 | e5.4.0) EOF } @@ -136,6 +137,12 @@ rel_branch() { e5.3.*) echo 'release-53' ;; + v5.4.*) + echo 'release-54' + ;; + e5.4.*) + echo 'release-54' + ;; *) logerr "Unsupported version tag $TAG" exit 1 diff --git a/scripts/rel/sync-remotes.sh b/scripts/rel/sync-remotes.sh index 9d3da2715..6b41415f0 100755 --- a/scripts/rel/sync-remotes.sh +++ b/scripts/rel/sync-remotes.sh @@ -5,7 +5,7 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." -BASE_BRANCHES=( 'release-53' 'release-52' 'release-51' 'master' ) +BASE_BRANCHES=( 'release-54' 'release-53' 'release-52' 'release-51' 'master' ) usage() { cat < Date: Mon, 4 Dec 2023 17:59:58 +0800 Subject: [PATCH 05/46] chore: upgrade dashboard to e1.4.0-beta.1 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 41812f6d9..45326a5e0 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 export EMQX_DASHBOARD_VERSION ?= v1.5.2 -export EMQX_EE_DASHBOARD_VERSION ?= e1.3.2 +export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0-beta.1 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise From ac618d331ffda6c88b91eaa3d94e815943742d26 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 1 Dec 2023 16:58:22 +0200 Subject: [PATCH 06/46] chore: upgrade grpc-erl to 0.6.12 grpc-erl 0.6.12 doesn't start dependent apps lazily anymore, as it was not deadlock safe. --- changes/ce/fix-12078.en.md | 3 +++ mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-12078.en.md diff --git a/changes/ce/fix-12078.en.md b/changes/ce/fix-12078.en.md new file mode 100644 index 000000000..3396e5a71 --- /dev/null +++ b/changes/ce/fix-12078.en.md @@ -0,0 +1,3 @@ +Upgrade grpc-erl to 0.6.12 + +grpc-erl 0.6.12 fixes a potential deadlock that was possible because grpc client started dependent apps lazily. diff --git a/mix.exs b/mix.exs index 747f5a205..159b105a5 100644 --- a/mix.exs +++ b/mix.exs @@ -57,7 +57,7 @@ defmodule EMQXUmbrella.MixProject do {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.2", override: true}, - {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, + {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.14", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, diff --git a/rebar.config b/rebar.config index e2cdf6018..eec2d505a 100644 --- a/rebar.config +++ b/rebar.config @@ -64,7 +64,7 @@ , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}} - , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} + , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.14"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} From e455433694eebc24f52aaf411b8a1a092ea4ca78 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 1 Dec 2023 17:55:33 +0200 Subject: [PATCH 07/46] test(emqx_gateway): fix test failures caused by grpc-erl upgrade --- .../test/emqx_bridge_hstreamdb_SUITE.erl | 21 ++++++++++--------- .../test/emqx_gateway_api_SUITE.erl | 4 ++-- .../test/emqx_gateway_authn_SUITE.erl | 6 ++++-- .../test/emqx_gateway_authz_SUITE.erl | 4 ++-- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 14ea202be..a9232d5fe 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -274,16 +274,17 @@ t_write_failure(Config) -> health_check_resource_down(Config), case QueryMode of sync -> - case EnableBatch of - true -> - %% append to batch always returns ok - ?assertMatch(ok, send_message(Config, Data)); - false -> - ?assertMatch( - {error, {cannot_list_shards, {<>, econnrefused}}}, - send_message(Config, Data) - ) - end; + %% Error (call timeout) is expected for both with_batch and without_batch. + %% `health_check_resource_down(Config)` above calls health check and asserts + %% that resource is already down. + %% So, emqx_resource_manager updates it state to disconnected before returning health_check result. + %% After that, emqx_resource_buffer_worker reads resource state and doesn't even attempt calling + %% hstreamdb connector, since it is disconnected, see: emqx_resource_buffer_worker.erl:1163: + %% ``` + %% do_call_query(_QM, _Id, _Index, _Ref, _Query, _QueryOpts, _Data) -> + %% ?RESOURCE_ERROR(not_connected, "resource not connected"). + %% ``` + ?assertMatch({error, _}, send_message(Config, Data)); async -> %% TODO: async mode is not supported yet, %% but it will return ok if calling emqx_resource_buffer_worker:async_query/3, diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 0b562e851..41409693a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -50,11 +50,11 @@ init_per_suite(Conf) -> emqx_config:delete_override_conf_files(), emqx_config:erase(gateway), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_auth, emqx_auth_mnesia, emqx_gateway]), + emqx_mgmt_api_test_util:init_suite([grpc, emqx_conf, emqx_auth, emqx_auth_mnesia, emqx_gateway]), Conf. end_per_suite(Conf) -> - emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auth_mnesia, emqx_auth, emqx_conf]), + emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_auth_mnesia, emqx_auth, emqx_conf, grpc]), Conf. init_per_testcase(t_gateway_fail, Config) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index f5e98bb37..0072447b6 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -69,7 +69,7 @@ init_per_suite(Config) -> emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:erase(gateway), init_gateway_conf(), - emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway]), + emqx_mgmt_api_test_util:init_suite([grpc, emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway]), application:ensure_all_started(cowboy), emqx_gateway_auth_ct:start(), timer:sleep(500), @@ -78,7 +78,9 @@ init_per_suite(Config) -> end_per_suite(Config) -> emqx_gateway_auth_ct:stop(), emqx_config:erase(gateway), - emqx_mgmt_api_test_util:end_suite([cowboy, emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway]), + emqx_mgmt_api_test_util:end_suite([ + cowboy, emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway, grpc + ]), Config. init_per_testcase(_Case, Config) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index d9d7167a9..dd149133b 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -69,7 +69,7 @@ init_per_suite(Config) -> emqx_gateway_test_utils:load_all_gateway_apps(), init_gateway_conf(), emqx_mgmt_api_test_util:init_suite([ - emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway + grpc, emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway ]), meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx_authz_file, create, fun(S) -> S end), @@ -83,7 +83,7 @@ end_per_suite(Config) -> ok = emqx_authz_test_lib:restore_authorizers(), emqx_config:erase(gateway), emqx_mgmt_api_test_util:end_suite([ - emqx_gateway, emqx_auth_http, emqx_auth, emqx_conf + emqx_gateway, emqx_auth_http, emqx_auth, emqx_conf, grpc ]), Config. From 4cbc0bdece94f706a58d86da2f5298755eed98e4 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 5 Dec 2023 17:23:57 +0800 Subject: [PATCH 08/46] build: bump oldest otp vsn to OTP-25 --- .gitignore | 4 ++++ rebar.config.erl | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0e7c614a7..6fd884057 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ _build .rebar3 rebar3.crashdump .DS_Store +# emqx_ds_replication_layer_meta:ensure_site/0 +data etc/gen.emqx.conf compile_commands.json cuttlefish @@ -69,3 +71,5 @@ bom.json ct_run*/ apps/emqx_conf/etc/emqx.conf.all.rendered* rebar-git-cache.tar +# build docker image locally +.docker_image_tag diff --git a/rebar.config.erl b/rebar.config.erl index e16be99d8..37f041bf6 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -15,7 +15,7 @@ do(Dir, CONFIG) -> end. assert_otp() -> - Oldest = 24, + Oldest = 25, Latest = 26, OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < Oldest orelse OtpRelease > Latest of From 1b68fbff05c54689e576170b498ee1da3a7bdf7c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 4 Dec 2023 16:44:01 -0300 Subject: [PATCH 09/46] fix(api): return list of dependent rule ids when trying to delete bridge/action Fixes https://emqx.atlassian.net/browse/EMQX-11523 --- apps/emqx_bridge/src/emqx_bridge_api.erl | 29 ++++++++++------- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 31 ++++++++++--------- .../test/emqx_bridge_api_SUITE.erl | 3 +- .../test/emqx_bridge_v2_api_SUITE.erl | 3 +- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 39 insertions(+), 31 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b725eb740..2343e3ccf 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -87,12 +87,15 @@ paths() -> "/bridges_probe" ]. -error_schema(Code, Message) when is_atom(Code) -> - error_schema([Code], Message); -error_schema(Codes, Message) when is_list(Message) -> - error_schema(Codes, list_to_binary(Message)); -error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) -> - emqx_dashboard_swagger:error_codes(Codes, Message). +error_schema(Code, Message) -> + error_schema(Code, Message, _ExtraFields = []). + +error_schema(Code, Message, ExtraFields) when is_atom(Code) -> + error_schema([Code], Message, ExtraFields); +error_schema(Codes, Message, ExtraFields) when is_list(Message) -> + error_schema(Codes, list_to_binary(Message), ExtraFields); +error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) -> + ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message). get_response_body_schema() -> emqx_dashboard_swagger:schema_with_examples( @@ -340,7 +343,8 @@ schema("/bridges/:id") -> 204 => <<"Bridge deleted">>, 400 => error_schema( 'BAD_REQUEST', - "Cannot delete bridge while active rules are defined for this bridge" + "Cannot delete bridge while active rules are defined for this bridge", + [{rules, mk(array(string()), #{desc => "Dependent Rule IDs"})}] ), 404 => error_schema('NOT_FOUND', "Bridge not found"), 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") @@ -517,11 +521,12 @@ schema("/bridges_probe") -> reason := rules_depending_on_this_bridge, rule_ids := RuleIds }} -> - RulesStr = [[" ", I] || I <- RuleIds], - Msg = bin([ - "Cannot delete bridge while active rules are depending on it:", RulesStr - ]), - ?BAD_REQUEST(Msg); + Msg0 = ?ERROR_MSG( + 'BAD_REQUEST', + bin("Cannot delete bridge while active rules are depending on it") + ), + Msg = Msg0#{rules => RuleIds}, + {400, Msg}; {error, timeout} -> ?SERVICE_UNAVAILABLE(<<"request timeout">>); {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index e6fcca50a..4cb371103 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -91,12 +91,15 @@ paths() -> "/action_types" ]. -error_schema(Code, Message) when is_atom(Code) -> - error_schema([Code], Message); -error_schema(Codes, Message) when is_list(Message) -> - error_schema(Codes, list_to_binary(Message)); -error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) -> - emqx_dashboard_swagger:error_codes(Codes, Message). +error_schema(Code, Message) -> + error_schema(Code, Message, _ExtraFields = []). + +error_schema(Code, Message, ExtraFields) when is_atom(Code) -> + error_schema([Code], Message, ExtraFields); +error_schema(Codes, Message, ExtraFields) when is_list(Message) -> + error_schema(Codes, list_to_binary(Message), ExtraFields); +error_schema(Codes, Message, ExtraFields) when is_list(Codes) andalso is_binary(Message) -> + ExtraFields ++ emqx_dashboard_swagger:error_codes(Codes, Message). get_response_body_schema() -> emqx_dashboard_swagger:schema_with_examples( @@ -247,7 +250,8 @@ schema("/actions/:id") -> 204 => <<"Bridge deleted">>, 400 => error_schema( 'BAD_REQUEST', - "Cannot delete bridge while active rules are defined for this bridge" + "Cannot delete bridge while active rules are defined for this bridge", + [{rules, mk(array(string()), #{desc => "Dependent Rule IDs"})}] ), 404 => error_schema('NOT_FOUND', "Bridge not found"), 503 => error_schema('SERVICE_UNAVAILABLE', "Service unavailable") @@ -445,15 +449,12 @@ schema("/action_types") -> reason := rules_depending_on_this_bridge, rule_ids := RuleIds }} -> - RuleIdLists = [binary_to_list(iolist_to_binary(X)) || X <- RuleIds], - RulesStr = string:join(RuleIdLists, ", "), - Msg = io_lib:format( - "Cannot delete bridge while active rules are depending on it: ~s\n" - "Append ?also_delete_dep_actions=true to the request URL to delete " - "rule actions that depend on this bridge as well.", - [RulesStr] + Msg0 = ?ERROR_MSG( + 'BAD_REQUEST', + bin("Cannot delete action while active rules are depending on it") ), - ?BAD_REQUEST(iolist_to_binary(Msg)); + Msg = Msg0#{rules => RuleIds}, + {400, Msg}; {error, timeout} -> ?SERVICE_UNAVAILABLE(<<"request timeout">>); {error, Reason} -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index e88206ccd..d54330d54 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -621,9 +621,10 @@ t_check_dependent_actions_on_delete(Config) -> Config ), %% deleting the bridge should fail because there is a rule that depends on it - {ok, 400, _} = request( + {ok, 400, Body} = request( delete, uri(["bridges", BridgeID]) ++ "?also_delete_dep_actions=false", Config ), + ?assertMatch(#{<<"rules">> := [_ | _]}, emqx_utils_json:decode(Body, [return_maps])), %% delete the rule first {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), Config), %% then delete the bridge is OK diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 83a857b47..16bbcf7a5 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -958,11 +958,12 @@ t_cascade_delete_actions(Config) -> }, Config ), - {ok, 400, _} = request( + {ok, 400, Body} = request( delete, uri([?ROOT, BridgeID]), Config ), + ?assertMatch(#{<<"rules">> := [_ | _]}, emqx_utils_json:decode(Body, [return_maps])), {ok, 200, [_]} = request_json(get, uri([?ROOT]), Config), %% Cleanup {ok, 204, _} = request( diff --git a/mix.exs b/mix.exs index 159b105a5..7768d84d3 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.2", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.14", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.15", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index eec2d505a..98a53ba3b 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.2"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.14"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.15"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} From e5a4be11f5896d80448dd0e3a4ecb3d6051bb319 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 5 Dec 2023 09:22:48 -0300 Subject: [PATCH 10/46] chore: stop downgrading action id in rule API responses Fixes https://emqx.atlassian.net/browse/EMQX-11405 --- .../src/emqx_rule_engine_api.erl | 22 +------------------ .../test/emqx_rule_engine_api_SUITE.erl | 15 +++++++------ 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b24662e53..246bd8f01 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -521,8 +521,7 @@ format_action(Actions) -> do_format_action({bridge, BridgeType, BridgeName, _ResId}) -> emqx_bridge_resource:bridge_id(BridgeType, BridgeName); -do_format_action({bridge_v2, BridgeType0, BridgeName}) -> - BridgeType = try_downgrade(BridgeType0, BridgeName), +do_format_action({bridge_v2, BridgeType, BridgeName}) -> emqx_bridge_resource:bridge_id(BridgeType, BridgeName); do_format_action(#{mod := Mod, func := Func, args := Args}) -> #{ @@ -534,25 +533,6 @@ do_format_action(#{mod := Mod, func := Func}) -> function => printable_function_name(Mod, Func) }. -try_downgrade(BridgeType, BridgeName) -> - Conf = try_get_conf(BridgeType, BridgeName), - try emqx_bridge_lib:downgrade_type(BridgeType, Conf) of - DowngradedBridgeType -> - DowngradedBridgeType - catch - error:{config_not_found, _} -> - BridgeType - end. - -try_get_conf(BridgeType, BridgeName) -> - try emqx_conf:get_raw([actions, BridgeType, BridgeName]) of - RawConf -> - RawConf - catch - error:{config_not_found, _} -> - #{} - end. - printable_function_name(emqx_rule_actions, Func) -> Func; printable_function_name(Mod, Func) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index c2c52b6a6..43a35f10e 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -310,25 +310,26 @@ t_rule_engine(_) -> }), {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). -t_downgrade_bridge_type(_) -> +t_dont_downgrade_bridge_type(_) -> case emqx_release:edition() of ee -> - do_test_downgrade_bridge_type(); + do_t_dont_downgrade_bridge_type(); ce -> %% downgrade is not supported in CE ok end. -do_test_downgrade_bridge_type() -> +do_t_dont_downgrade_bridge_type() -> + %% Create a rule using a bridge V1 ID #{id := RuleId} = create_rule((?SIMPLE_RULE(<<>>))#{<<"actions">> => [<<"kafka:name">>]}), ?assertMatch( - %% returns a bridges_v2 ID - {200, #{data := [#{actions := [<<"kafka:name">>]}]}}, + %% returns an action ID + {200, #{data := [#{actions := [<<"kafka_producer:name">>]}]}}, emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}) ), ?assertMatch( - %% returns a bridges_v2 ID - {200, #{actions := [<<"kafka:name">>]}}, + %% returns an action ID + {200, #{actions := [<<"kafka_producer:name">>]}}, emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) ), ok. From b2ca59dfa718c6669141c6c8e5418fa68c0a17ee Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:46:27 +0100 Subject: [PATCH 11/46] fix(bridge_gcp): Don't print config to the logs --- .../src/emqx_bridge_gcp_pubsub_impl_producer.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index cd7568001..fbecb14fa 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -58,7 +58,7 @@ query_mode(_Config) -> async. on_start(InstanceId, Config0) -> ?SLOG(info, #{ msg => "starting_gcp_pubsub_bridge", - config => Config0 + instance_id => InstanceId }), Config = maps:update_with(service_account_json, fun emqx_utils_maps:binary_key_map/1, Config0), #{ From e42f4155d3ba016e529c1e7a698c3794415e1be3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 5 Dec 2023 10:56:29 -0300 Subject: [PATCH 12/46] fix(bridges/actions api): correctly deobfuscate passwords when probing Fixes https://emqx.atlassian.net/browse/EMQX-11533 --- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 4 +- .../src/schema/emqx_bridge_schema.erl | 5 +- ...qx_bridge_v1_compatibility_layer_SUITE.erl | 94 +++++++++++++++++++ apps/emqx_utils/src/emqx_utils.erl | 3 + apps/emqx_utils/test/emqx_utils_tests.erl | 28 ++++++ 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_utils/test/emqx_utils_tests.erl diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index e6fcca50a..93121f98b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -550,8 +550,8 @@ schema("/action_types") -> '/action_types'(get, _Request) -> ?OK(emqx_bridge_v2_schema:types()). -maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType, <<"name">> := BridgeName} = Params) -> - case emqx_bridge:lookup(BridgeType, BridgeName) of +maybe_deobfuscate_bridge_probe(#{<<"type">> := ActionType, <<"name">> := BridgeName} = Params) -> + case emqx_bridge_v2:lookup(ActionType, BridgeName) of {ok, #{raw_config := RawConf}} -> %% TODO check if RawConf optained above is compatible with the commented out code below %% RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 27b3a8f14..1c4d5365d 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -35,6 +35,9 @@ metrics_fields/0 ]). +%% for testing only +-export([enterprise_api_schemas/1]). + %%====================================================================================== %% Hocon Schema Definitions @@ -57,7 +60,7 @@ api_schema(Method) -> {<<"mqtt">>, emqx_bridge_mqtt_schema} ] ], - EE = enterprise_api_schemas(Method), + EE = ?MODULE:enterprise_api_schemas(Method), hoconsc:union(bridge_api_union(Broker ++ EE)). bridge_api_union(Refs) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl index b5c0ec9f2..215013a6d 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v1_compatibility_layer_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/asserts.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -108,6 +109,14 @@ setup_mocks() -> fun(Method) -> [{bridge_type_bin(), hoconsc:ref(?MODULE, "api_" ++ Method)}] end ), + catch meck:new(emqx_bridge_schema, MeckOpts), + meck:expect( + emqx_bridge_schema, + enterprise_api_schemas, + 1, + fun(Method) -> [{bridge_type_bin(), hoconsc:ref(?MODULE, "api_" ++ Method)}] end + ), + ok. con_mod() -> @@ -142,7 +151,9 @@ con_schema() -> fields("connector") -> [ {enable, hoconsc:mk(any(), #{})}, + {password, emqx_schema_secret:mk(#{required => false})}, {resource_opts, hoconsc:mk(map(), #{})}, + {on_start_fun, hoconsc:mk(binary(), #{})}, {ssl, hoconsc:ref(ssl)} ]; fields("api_post") -> @@ -458,6 +469,29 @@ enable_rule_http(RuleId) -> Params = #{<<"enable">> => true}, update_rule_http(RuleId, Params). +probe_bridge_http_api_v1(Opts) -> + Name = maps:get(name, Opts), + Overrides = maps:get(overrides, Opts, #{}), + BridgeConfig0 = emqx_utils_maps:deep_merge(bridge_config(), Overrides), + BridgeConfig = maps:without([<<"connector">>], BridgeConfig0), + Params = BridgeConfig#{<<"type">> => bridge_type_bin(), <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["bridges_probe"]), + ct:pal("probe bridge (http v1) (~p):\n ~p", [#{name => Name}, Params]), + Res = request(post, Path, Params), + ct:pal("probe bridge (http v1) (~p) result:\n ~p", [#{name => Name}, Res]), + Res. + +probe_action_http_api_v2(Opts) -> + Name = maps:get(name, Opts), + Overrides = maps:get(overrides, Opts, #{}), + BridgeConfig = emqx_utils_maps:deep_merge(bridge_config(), Overrides), + Params = BridgeConfig#{<<"type">> => bridge_type_bin(), <<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path(["actions_probe"]), + ct:pal("probe action (http v2) (~p):\n ~p", [#{name => Name}, Params]), + Res = request(post, Path, Params), + ct:pal("probe action (http v2) (~p) result:\n ~p", [#{name => Name}, Res]), + Res. + %%------------------------------------------------------------------------------ %% Test cases %%------------------------------------------------------------------------------ @@ -825,3 +859,63 @@ t_create_with_bad_name(_Config) -> } }}} = create_bridge_http_api_v1(Opts), ok. + +t_obfuscated_secrets_probe(_Config) -> + Name = <<"bridgev2">>, + Me = self(), + ets:new(emqx_bridge_v2_SUITE:fun_table_name(), [named_table, public]), + OnStartFun = emqx_bridge_v2_SUITE:wrap_fun(fun(Conf) -> + Me ! {on_start, Conf}, + {ok, Conf} + end), + OriginalPassword = <<"supersecret">>, + Overrides = #{<<"password">> => OriginalPassword, <<"on_start_fun">> => OnStartFun}, + %% Using the real password, like when creating the bridge for the first time. + ?assertMatch( + {ok, {{_, 204, _}, _, _}}, + probe_bridge_http_api_v1(#{name => Name, overrides => Overrides}) + ), + + %% Check that we still can probe created bridges that use passwords. + ?assertMatch( + {ok, {{_, 201, _}, _, #{}}}, + create_bridge_http_api_v1(#{name => Name, overrides => Overrides}) + ), + %% Password is obfuscated + ?assertMatch( + {ok, {{_, 200, _}, _, #{<<"password">> := <<"******">>}}}, + get_bridge_http_api_v1(Name) + ), + %% still using the password + ?assertMatch( + {ok, {{_, 204, _}, _, _}}, + probe_bridge_http_api_v1(#{name => Name, overrides => Overrides}) + ), + %% now with obfuscated password (loading the UI again) + ?assertMatch( + {ok, {{_, 204, _}, _, _}}, + probe_bridge_http_api_v1(#{ + name => Name, + overrides => Overrides#{<<"password">> => <<"******">>} + }) + ), + ?assertMatch( + {ok, {{_, 204, _}, _, _}}, + probe_action_http_api_v2(#{ + name => Name, + overrides => Overrides#{<<"password">> => <<"******">>} + }) + ), + + %% We have to check that the connector was started with real passwords during dry runs + StartConfs = [Conf || {on_start, Conf} <- ?drainMailbox()], + Passwords = lists:map(fun(#{password := P}) -> P end, StartConfs), + ?assert(lists:all(fun is_function/1, Passwords), #{passwords => Passwords}), + UnwrappedPasswords = [F() || F <- Passwords], + ?assertEqual( + [OriginalPassword], + lists:usort(UnwrappedPasswords), + #{passwords => UnwrappedPasswords} + ), + + ok. diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index f827f65de..7625d5ca7 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -759,6 +759,9 @@ do_is_redacted(K, ?REDACT_VAL, Fun) -> Fun(K); do_is_redacted(K, <>, Fun) -> Fun(K); +do_is_redacted(_K, V, _Fun) when is_function(V, 0) -> + %% already wrapped by `emqx_secret' or other module + true; do_is_redacted(_K, _V, _Fun) -> false. diff --git a/apps/emqx_utils/test/emqx_utils_tests.erl b/apps/emqx_utils/test/emqx_utils_tests.erl new file mode 100644 index 000000000..b72ce169e --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_tests.erl @@ -0,0 +1,28 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_utils_tests). + +-include_lib("eunit/include/eunit.hrl"). + +is_redacted_test_() -> + [ + ?_assertNot(emqx_utils:is_redacted(password, <<"secretpass">>)), + ?_assertNot(emqx_utils:is_redacted(password, <<>>)), + ?_assertNot(emqx_utils:is_redacted(password, undefined)), + ?_assert(emqx_utils:is_redacted(password, <<"******">>)), + ?_assert(emqx_utils:is_redacted(password, fun() -> <<"secretpass">> end)), + ?_assert(emqx_utils:is_redacted(password, emqx_secret:wrap(<<"secretpass">>))) + ]. From 7fdc650448f7173204d98b0978c7186dd542ad47 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Mon, 20 Nov 2023 17:05:10 +0200 Subject: [PATCH 13/46] feat: integrate OpenTelemetry traces --- apps/emqx/src/emqx_channel.erl | 12 +- apps/emqx/src/emqx_connection.erl | 9 +- apps/emqx/src/emqx_external_trace.erl | 109 +++++++ apps/emqx/src/emqx_frame.erl | 2 + apps/emqx/src/emqx_message.erl | 12 +- apps/emqx/src/emqx_packet.erl | 8 +- apps/emqx/test/emqx_message_SUITE.erl | 4 +- apps/emqx_opentelemetry/src/emqx_otel_app.erl | 2 + .../src/emqx_otel_config.erl | 64 ++-- .../src/emqx_otel_schema.erl | 82 +++++- .../src/emqx_otel_trace.erl | 277 ++++++++++++++++++ apps/emqx_utils/include/emqx_message.hrl | 4 +- changes/ce/feat-11984.en.md | 1 + rel/i18n/emqx_otel_schema.hocon | 12 + 14 files changed, 566 insertions(+), 32 deletions(-) create mode 100644 apps/emqx/src/emqx_external_trace.erl create mode 100644 apps/emqx_opentelemetry/src/emqx_otel_trace.erl create mode 100644 changes/ce/feat-11984.en.md diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 816ab7b2b..686d524dc 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -398,8 +398,10 @@ handle_in(?PACKET(_), Channel = #channel{conn_state = ConnState}) when handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel); handle_in(Packet = ?PUBLISH_PACKET(_QoS), Channel) -> case emqx_packet:check(Packet) of - ok -> process_publish(Packet, Channel); - {error, ReasonCode} -> handle_out(disconnect, ReasonCode, Channel) + ok -> + emqx_external_trace:trace_process_publish(Packet, Channel, fun process_publish/2); + {error, ReasonCode} -> + handle_out(disconnect, ReasonCode, Channel) end; handle_in( ?PUBACK_PACKET(PacketId, _ReasonCode, Properties), @@ -921,7 +923,11 @@ handle_deliver( Messages = emqx_session:enrich_delivers(ClientInfo, Delivers1, Session), NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session), {ok, Channel#channel{session = NSession}}; -handle_deliver( +handle_deliver(Delivers, Channel) -> + Delivers1 = emqx_external_trace:start_trace_send(Delivers, Channel), + do_handle_deliver(Delivers1, Channel). + +do_handle_deliver( Delivers, Channel = #channel{ session = Session, diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 11d42f9dd..66160ed36 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -855,9 +855,14 @@ with_channel(Fun, Args, State = #state{channel = Channel}) -> %%-------------------------------------------------------------------- %% Handle outgoing packets -handle_outgoing(Packets, State) when is_list(Packets) -> +handle_outgoing(Packets, State) -> + Res = do_handle_outgoing(Packets, State), + emqx_external_trace:end_trace_send(Packets), + Res. + +do_handle_outgoing(Packets, State) when is_list(Packets) -> send(lists:map(serialize_and_inc_stats_fun(State), Packets), State); -handle_outgoing(Packet, State) -> +do_handle_outgoing(Packet, State) -> send((serialize_and_inc_stats_fun(State))(Packet), State). serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> diff --git a/apps/emqx/src/emqx_external_trace.erl b/apps/emqx/src/emqx_external_trace.erl new file mode 100644 index 000000000..7f8823903 --- /dev/null +++ b/apps/emqx/src/emqx_external_trace.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_external_trace). + +-callback trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when + Packet :: emqx_types:packet(), + Channel :: emqx_channel:channel(), + Res :: term(). + +-callback start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> + list(emqx_types:deliver()). + +-callback end_trace_send(emqx_types:packet() | [emqx_types:packet()]) -> ok. + +-callback event(EventName :: term(), Attributes :: term()) -> ok. + +-export([ + register_provider/1, + unregister_provider/1, + trace_process_publish/3, + start_trace_send/2, + end_trace_send/1, + event/1, + event/2 +]). + +-define(PROVIDER, {?MODULE, trace_provider}). + +-define(with_provider(IfRegisitered, IfNotRegisired), + case persistent_term:get(?PROVIDER, undefined) of + undefined -> + IfNotRegisired; + Provider -> + Provider:IfRegisitered + end +). + +%%-------------------------------------------------------------------- +%% provider API +%%-------------------------------------------------------------------- + +-spec register_provider(module()) -> ok | {error, term()}. +register_provider(Module) when is_atom(Module) -> + case is_valid_provider(Module) of + true -> + persistent_term:put(?PROVIDER, Module); + false -> + {error, invalid_provider} + end. + +-spec unregister_provider(module()) -> ok | {error, term()}. +unregister_provider(Module) -> + case persistent_term:get(?PROVIDER, undefined) of + Module -> + persistent_term:erase(?PROVIDER), + ok; + _ -> + {error, not_registered} + end. + +%%-------------------------------------------------------------------- +%% trace API +%%-------------------------------------------------------------------- + +-spec trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when + Packet :: emqx_types:packet(), + Channel :: emqx_channel:channel(), + Res :: term(). +trace_process_publish(Packet, Channel, ProcessFun) -> + ?with_provider(?FUNCTION_NAME(Packet, Channel, ProcessFun), ProcessFun(Packet, Channel)). + +-spec start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> + list(emqx_types:deliver()). +start_trace_send(Delivers, Channel) -> + ?with_provider(?FUNCTION_NAME(Delivers, Channel), Delivers). + +-spec end_trace_send(emqx_types:packet() | [emqx_types:packet()]) -> ok. +end_trace_send(Packets) -> + ?with_provider(?FUNCTION_NAME(Packets), ok). + +event(Name) -> + event(Name, #{}). + +-spec event(term(), term()) -> ok. +event(Name, Attributes) -> + ?with_provider(?FUNCTION_NAME(Name, Attributes), ok). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +is_valid_provider(Module) -> + lists:all( + fun({F, A}) -> erlang:function_exported(Module, F, A) end, + ?MODULE:behaviour_info(callbacks) + ). diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 20be12c42..0799a24ee 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -960,6 +960,8 @@ serialize_properties(Props) when is_map(Props) -> serialize_property(_, Disabled) when Disabled =:= disabled; Disabled =:= undefined -> <<>>; +serialize_property(internal_extra, _) -> + <<>>; serialize_property('Payload-Format-Indicator', Val) -> <<16#01, Val>>; serialize_property('Message-Expiry-Interval', Val) -> diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index b65c8360f..6b684c199 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -311,7 +311,8 @@ to_packet( qos = QoS, headers = Headers, topic = Topic, - payload = Payload + payload = Payload, + extra = Extra } ) -> #mqtt_packet{ @@ -324,8 +325,8 @@ to_packet( variable = #mqtt_packet_publish{ topic_name = Topic, packet_id = PacketId, - properties = filter_pub_props( - maps:get(properties, Headers, #{}) + properties = maybe_put_extra( + Extra, filter_pub_props(maps:get(properties, Headers, #{})) ) }, payload = Payload @@ -345,6 +346,11 @@ filter_pub_props(Props) -> Props ). +maybe_put_extra(Extra, Props) when map_size(Extra) > 0 -> + Props#{internal_extra => Extra}; +maybe_put_extra(_Extra, Props) -> + Props. + %% @doc Message to map -spec to_map(emqx_types:message()) -> message_map(). to_map(#message{ diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index 9cb23be2e..542dc8b3b 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -452,9 +452,15 @@ to_message( Headers ) -> Msg = emqx_message:make(ClientId, QoS, Topic, Payload), + {Extra, Props1} = + case maps:take(internal_extra, Props) of + error -> {#{}, Props}; + ExtraProps -> ExtraProps + end, Msg#message{ flags = #{dup => Dup, retain => Retain}, - headers = Headers#{properties => Props} + headers = Headers#{properties => Props1}, + extra = Extra }. -spec will_msg(#mqtt_packet_connect{}) -> emqx_types:message(). diff --git a/apps/emqx/test/emqx_message_SUITE.erl b/apps/emqx/test/emqx_message_SUITE.erl index f404ff15d..2e4164652 100644 --- a/apps/emqx/test/emqx_message_SUITE.erl +++ b/apps/emqx/test/emqx_message_SUITE.erl @@ -207,7 +207,7 @@ t_to_map(_) -> {topic, <<"topic">>}, {payload, <<"payload">>}, {timestamp, emqx_message:timestamp(Msg)}, - {extra, []} + {extra, #{}} ], ?assertEqual(List, emqx_message:to_list(Msg)), ?assertEqual(maps:from_list(List), emqx_message:to_map(Msg)). @@ -223,7 +223,7 @@ t_from_map(_) -> topic => <<"topic">>, payload => <<"payload">>, timestamp => emqx_message:timestamp(Msg), - extra => [] + extra => #{} }, ?assertEqual(Map, emqx_message:to_map(Msg)), ?assertEqual(Msg, emqx_message:from_map(emqx_message:to_map(Msg))). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_app.erl b/apps/emqx_opentelemetry/src/emqx_otel_app.erl index cf93d7753..f4b579fe5 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_app.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_app.erl @@ -24,10 +24,12 @@ start(_StartType, _StartArgs) -> emqx_otel_config:add_handler(), ok = emqx_otel_config:add_otel_log_handler(), + ok = emqx_otel_trace:ensure_traces(emqx:get_config([opentelemetry, traces])), emqx_otel_sup:start_link(). stop(_State) -> emqx_otel_config:remove_handler(), + _ = emqx_otel_trace:stop(), _ = emqx_otel_config:remove_otel_log_handler(), ok. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index 11e97dcdd..d2cc81521 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -54,27 +54,24 @@ remove_handler() -> post_config_update(?OPTL, _Req, Old, Old, _AppEnvs) -> ok; -post_config_update(?OPTL, _Req, New, _Old, AppEnvs) -> +post_config_update(?OPTL, _Req, New, Old, AppEnvs) -> application:set_env(AppEnvs), - MetricsRes = ensure_otel_metrics(New), - LogsRes = ensure_otel_logs(New), + MetricsRes = ensure_otel_metrics(New, Old), + LogsRes = ensure_otel_logs(New, Old), + TracesRes = ensure_otel_traces(New, Old), _ = maybe_stop_all_otel_apps(New), - case {MetricsRes, LogsRes} of - {ok, ok} -> ok; + case {MetricsRes, LogsRes, TracesRes} of + {ok, ok, ok} -> ok; Other -> {error, Other} end; post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. stop_all_otel_apps() -> - _ = application:stop(opentelemetry), - _ = application:stop(opentelemetry_experimental), - _ = application:stop(opentelemetry_experimental_api), - _ = application:stop(opentelemetry_exporter), - ok. + stop_all_otel_apps(true). add_otel_log_handler() -> - ensure_otel_logs(emqx:get_config(?OPTL)). + ensure_otel_logs(emqx:get_config(?OPTL), #{}). remove_otel_log_handler() -> remove_handler_if_present(?OTEL_LOG_HANDLER_ID). @@ -93,23 +90,34 @@ otel_exporter(ExporterConf) -> %% Internal functions -ensure_otel_metrics(#{metrics := #{enable := true} = MetricsConf}) -> +ensure_otel_metrics(#{metrics := MetricsConf}, #{metrics := MetricsConf}) -> + ok; +ensure_otel_metrics(#{metrics := #{enable := true} = MetricsConf}, _Old) -> _ = emqx_otel_metrics:stop_otel(), emqx_otel_metrics:start_otel(MetricsConf); -ensure_otel_metrics(#{metrics := #{enable := false}}) -> +ensure_otel_metrics(#{metrics := #{enable := false}}, _Old) -> emqx_otel_metrics:stop_otel(); -ensure_otel_metrics(_) -> +ensure_otel_metrics(_, _) -> ok. -ensure_otel_logs(#{logs := #{enable := true} = LogsConf}) -> +ensure_otel_logs(#{logs := LogsConf}, #{logs := LogsConf}) -> + ok; +ensure_otel_logs(#{logs := #{enable := true} = LogsConf}, _OldConf) -> ok = remove_handler_if_present(?OTEL_LOG_HANDLER_ID), ok = ensure_log_apps(), HandlerConf = tr_handler_conf(LogsConf), %% NOTE: should primary logger level be updated if it's higher than otel log level? logger:add_handler(?OTEL_LOG_HANDLER_ID, ?OTEL_LOG_HANDLER, HandlerConf); -ensure_otel_logs(#{logs := #{enable := false}}) -> +ensure_otel_logs(#{logs := #{enable := false}}, _OldConf) -> remove_handler_if_present(?OTEL_LOG_HANDLER_ID). +ensure_otel_traces(#{traces := TracesConf}, #{traces := TracesConf}) -> + ok; +ensure_otel_traces(#{traces := #{enable := true} = TracesConf}, _OldConf) -> + emqx_otel_trace:start(TracesConf); +ensure_otel_traces(#{traces := #{enable := false}}, _OldConf) -> + emqx_otel_trace:stop(). + remove_handler_if_present(HandlerId) -> case logger:get_handler_config(HandlerId) of {ok, _} -> @@ -123,8 +131,13 @@ ensure_log_apps() -> {ok, _} = application:ensure_all_started(opentelemetry_experimental), ok. -maybe_stop_all_otel_apps(#{metrics := #{enable := false}, logs := #{enable := false}}) -> - stop_all_otel_apps(); +maybe_stop_all_otel_apps(#{ + metrics := #{enable := false}, + logs := #{enable := false}, + traces := #{enable := false} +}) -> + IsShutdown = false, + stop_all_otel_apps(IsShutdown); maybe_stop_all_otel_apps(_) -> ok. @@ -158,3 +171,18 @@ is_ssl(<<"https://", _/binary>> = _Endpoint) -> true; is_ssl(_Endpoint) -> false. + +stop_all_otel_apps(IsShutdown) -> + %% if traces were enabled, it's not safe to stop opentelemetry app, + %% as there could be not finsihed traces that would crash if spans ETS tables are deleted + _ = + case IsShutdown of + true -> + _ = application:stop(opentelemetry); + false -> + ok + end, + _ = application:stop(opentelemetry_experimental), + _ = application:stop(opentelemetry_experimental_api), + _ = application:stop(opentelemetry_exporter), + ok. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index 927bc9dfd..6359f88a5 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -62,6 +62,13 @@ fields("opentelemetry") -> #{ desc => ?DESC(otel_logs) } + )}, + {traces, + ?HOCON( + ?R_REF("otel_traces"), + #{ + desc => ?DESC(otel_traces) + } )} ]; fields("otel_metrics") -> @@ -137,21 +144,94 @@ fields("otel_logs") -> } )} ]; +fields("otel_traces") -> + [ + {enable, + ?HOCON( + boolean(), + #{ + default => false, + desc => ?DESC(enable), + importance => ?IMPORTANCE_HIGH + } + )}, + {max_queue_size, + ?HOCON( + pos_integer(), + #{ + default => 2048, + desc => ?DESC(max_queue_size), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {exporting_timeout, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"30s">>, + desc => ?DESC(exporting_timeout), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {scheduled_delay, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + default => <<"5s">>, + desc => ?DESC(scheduled_delay), + importance => ?IMPORTANCE_HIDDEN + } + )}, + {exporter, + ?HOCON( + ?R_REF("otel_traces_exporter"), + #{ + desc => ?DESC(exporter), + importance => ?IMPORTANCE_HIGH + } + )}, + {filter, + ?HOCON( + ?R_REF("trace_filter"), + #{ + desc => ?DESC(trace_filter), + importance => ?IMPORTANCE_MEDIUM + } + )} + ]; fields("otel_metrics_exporter") -> exporter_fields(metrics); fields("otel_logs_exporter") -> exporter_fields(logs); fields("ssl_opts") -> Schema = emqx_schema:client_ssl_opts_schema(#{}), - lists:keydelete("enable", 1, Schema). + lists:keydelete("enable", 1, Schema); +fields("otel_traces_exporter") -> + exporter_fields(traces); +fields("trace_filter") -> + %% More filters can be implemented in future, e.g. topic, clientid + [ + {trace_all, + ?HOCON( + boolean(), + #{ + default => false, + desc => ?DESC(trace_all), + importance => ?IMPORTANCE_MEDIUM + } + )} + ]. desc("opentelemetry") -> ?DESC(opentelemetry); desc("exporter") -> ?DESC(exporter); desc("otel_logs_exporter") -> ?DESC(exporter); desc("otel_metrics_exporter") -> ?DESC(exporter); +desc("otel_traces_exporter") -> ?DESC(exporter); desc("otel_logs") -> ?DESC(otel_logs); desc("otel_metrics") -> ?DESC(otel_metrics); +desc("otel_traces") -> ?DESC(otel_traces); desc("ssl_opts") -> ?DESC(exporter_ssl); +desc("trace_filter") -> ?DESC(trace_filter); desc(_) -> undefined. exporter_fields(OtelSignal) -> diff --git a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl new file mode 100644 index 000000000..0c78f0abd --- /dev/null +++ b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl @@ -0,0 +1,277 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_trace). + +-behaviour(emqx_external_trace). + +-export([ + ensure_traces/1, + start/1, + stop/0 +]). + +-export([toggle_registered/1]). + +-export([ + trace_process_publish/3, + start_trace_send/2, + end_trace_send/1, + event/2 +]). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("opentelemetry_api/include/otel_tracer.hrl"). + +-define(EMQX_OTEL_CTX, otel_ctx). +%% NOTE: it's possible to use trace_flags to set is_sampled flag +-define(IS_ENABLED, emqx_enable). +-define(USER_PROPERTY, 'User-Property'). + +-define(TRACE_ALL_KEY, {?MODULE, trace_all}). +-define(TRACE_ALL, persistent_term:get(?TRACE_ALL_KEY, false)). + +%%-------------------------------------------------------------------- +%% config +%%-------------------------------------------------------------------- + +-spec toggle_registered(boolean()) -> ok | {error, term()}. +toggle_registered(true = _Enable) -> + emqx_external_trace:register_provider(?MODULE); +toggle_registered(false = _Enable) -> + _ = emqx_external_trace:unregister_provider(?MODULE), + ok. + +-spec ensure_traces(map()) -> ok | {error, term()}. +ensure_traces(#{enable := true} = Conf) -> + start(Conf); +ensure_traces(_Conf) -> + ok. + +-spec start(map()) -> ok | {error, term()}. +start(Conf) -> + _ = safe_stop_default_tracer(), + #{ + exporter := Exporter, + max_queue_size := MaxQueueSize, + exporting_timeout := ExportingTimeout, + scheduled_delay := ScheduledDelay, + filter := #{trace_all := TraceAll} + } = Conf, + OtelEnv = [ + {bsp_scheduled_delay_ms, ScheduledDelay}, + {bsp_exporting_timeout_ms, ExportingTimeout}, + {bsp_max_queue_size, MaxQueueSize}, + {traces_exporter, emqx_otel_config:otel_exporter(Exporter)} + ], + set_trace_all(TraceAll), + ok = application:set_env([{opentelemetry, OtelEnv}]), + _ = application:ensure_all_started(opentelemetry), + Res = assert_started(opentelemetry:start_default_tracer_provider()), + case Res of + ok -> + _ = toggle_registered(true), + Res; + Err -> + Err + end. + +-spec stop() -> ok. +stop() -> + _ = toggle_registered(false), + safe_stop_default_tracer(). + +%%-------------------------------------------------------------------- +%% trace API +%%-------------------------------------------------------------------- + +-spec trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when + Packet :: emqx_types:packet(), + Channel :: emqx_channel:channel(), + Res :: term(). +trace_process_publish(Packet, Channel, ProcessFun) -> + case maybe_init_ctx(Packet) of + false -> + ProcessFun(Packet, Channel); + RootCtx -> + RootCtx1 = otel_ctx:set_value(RootCtx, ?IS_ENABLED, true), + Attrs = maps:merge(packet_attributes(Packet), channel_attributes(Channel)), + SpanCtx = otel_tracer:start_span(RootCtx1, ?current_tracer, process_message, #{ + attributes => Attrs + }), + Ctx = otel_tracer:set_current_span(RootCtx1, SpanCtx), + %% put ctx to packet, so it can be further propagated + Packet1 = put_ctx_to_packet(Ctx, Packet), + %% TODO: consider getting rid of propagating Ctx through process dict as it's anyway seems to have + %% very limited usage + otel_ctx:attach(Ctx), + try + ProcessFun(Packet1, Channel) + after + ?end_span(), + clear() + end + end. + +-spec start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> + list(emqx_types:deliver()). +start_trace_send(Delivers, Channel) -> + lists:map( + fun({deliver, Topic, Msg} = Deliver) -> + case get_ctx_from_msg(Msg) of + Ctx when is_map(Ctx) -> + Attrs = maps:merge( + msg_attributes(Msg), sub_channel_attributes(Channel) + ), + StartOpts = #{attributes => Attrs}, + SpanCtx = otel_tracer:start_span( + Ctx, ?current_tracer, send_published_message, StartOpts + ), + Msg1 = put_ctx_to_msg( + otel_tracer:set_current_span(Ctx, SpanCtx), Msg + ), + {deliver, Topic, Msg1}; + _ -> + Deliver + end + end, + Delivers + ). + +-spec end_trace_send(emqx_types:packet() | [emqx_types:packet()]) -> ok. +end_trace_send(Packets) -> + lists:foreach( + fun(Packet) -> + case get_ctx_from_packet(Packet) of + Ctx when is_map(Ctx) -> + otel_span:end_span(otel_tracer:current_span_ctx(Ctx)); + _ -> + ok + end + end, + packets_list(Packets) + ). + +%% NOTE: adds an event only within an active span (Otel Ctx must be set in the calling process dict) +-spec event(opentelemetry:event_name(), opentelemetry:attributes_map()) -> ok. +event(Name, Attributes) -> + case otel_ctx:get_value(?IS_ENABLED, false) of + true -> + ?add_event(Name, Attributes), + ok; + false -> + ok + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +packets_list(Packets) when is_list(Packets) -> + Packets; +packets_list(Packet) -> + [Packet]. + +maybe_init_ctx(#mqtt_packet{variable = Packet}) -> + case should_trace_packet(Packet) of + true -> + Ctx = extract_traceparent_from_packet(Packet), + should_trace_context(Ctx) andalso Ctx; + false -> + false + end. + +extract_traceparent_from_packet(Packet) -> + Ctx = otel_ctx:new(), + case emqx_packet:info(properties, Packet) of + #{?USER_PROPERTY := UserProps} -> + otel_propagator_text_map:extract_to(Ctx, UserProps); + _ -> + Ctx + end. + +should_trace_context(RootCtx) -> + map_size(RootCtx) > 0 orelse ?TRACE_ALL. + +should_trace_packet(Packet) -> + not is_sys(emqx_packet:info(topic_name, Packet)). + +%% TODO: move to emqx_topic module? +is_sys(<<"$SYS/", _/binary>> = _Topic) -> true; +is_sys(_Topic) -> false. + +msg_attributes(Msg) -> + #{ + 'messaging.destination.name' => emqx_message:topic(Msg), + 'messaging.client_id' => emqx_message:from(Msg) + }. + +packet_attributes(#mqtt_packet{variable = Packet}) -> + #{'messaging.destination.name' => emqx_packet:info(topic_name, Packet)}. + +channel_attributes(Channel) -> + #{'messaging.client_id' => emqx_channel:info(clientid, Channel)}. + +sub_channel_attributes(Channel) -> + channel_attributes(Channel). + +put_ctx_to_msg(OtelCtx, Msg = #message{extra = Extra}) when is_map(Extra) -> + Msg#message{extra = Extra#{?EMQX_OTEL_CTX => OtelCtx}}; +%% extra field has not being used previously and defaulted to an empty list, it's safe to overwrite it +put_ctx_to_msg(OtelCtx, Msg) when is_record(Msg, message) -> + Msg#message{extra = #{?EMQX_OTEL_CTX => OtelCtx}}. + +put_ctx_to_packet( + OtelCtx, #mqtt_packet{variable = #mqtt_packet_publish{properties = Props} = PubPacket} = Packet +) -> + Extra = maps:get(internal_extra, Props, #{}), + Props1 = Props#{internal_extra => Extra#{?EMQX_OTEL_CTX => OtelCtx}}, + Packet#mqtt_packet{variable = PubPacket#mqtt_packet_publish{properties = Props1}}. + +get_ctx_from_msg(#message{extra = Extra}) -> + from_extra(Extra). + +get_ctx_from_packet(#mqtt_packet{ + variable = #mqtt_packet_publish{properties = #{internal_extra := Extra}} +}) -> + from_extra(Extra); +get_ctx_from_packet(_) -> + undefined. + +from_extra(#{?EMQX_OTEL_CTX := OtelCtx}) -> + OtelCtx; +from_extra(_) -> + undefined. + +clear() -> + otel_ctx:clear(). + +safe_stop_default_tracer() -> + try + _ = opentelemetry:stop_default_tracer_provider() + catch + %% noramal scenario, opentelemetry supervisor is not started + exit:{noproc, _} -> ok + end, + ok. + +assert_started({ok, _Pid}) -> ok; +assert_started({ok, _Pid, _Info}) -> ok; +assert_started({error, {already_started, _Pid}}) -> ok; +assert_started({error, Reason}) -> {error, Reason}. + +set_trace_all(TraceAll) -> + persistent_term:put({?MODULE, trace_all}, TraceAll). diff --git a/apps/emqx_utils/include/emqx_message.hrl b/apps/emqx_utils/include/emqx_message.hrl index a0d196fa9..cbb452c41 100644 --- a/apps/emqx_utils/include/emqx_message.hrl +++ b/apps/emqx_utils/include/emqx_message.hrl @@ -36,8 +36,8 @@ payload :: emqx_types:payload(), %% Timestamp (Unit: millisecond) timestamp :: integer(), - %% not used so far, for future extension - extra = [] :: term() + %% Miscellaneous extensions, currently used for OpenTelemetry context propagation + extra = #{} :: term() }). -endif. diff --git a/changes/ce/feat-11984.en.md b/changes/ce/feat-11984.en.md new file mode 100644 index 000000000..e4e0a7717 --- /dev/null +++ b/changes/ce/feat-11984.en.md @@ -0,0 +1 @@ +Implemented Open Telemetry distributed tracing feature. diff --git a/rel/i18n/emqx_otel_schema.hocon b/rel/i18n/emqx_otel_schema.hocon index 9e59b2a76..0a41874b9 100644 --- a/rel/i18n/emqx_otel_schema.hocon +++ b/rel/i18n/emqx_otel_schema.hocon @@ -11,6 +11,9 @@ otel_logs.label: "Open Telemetry Logs" otel_metrics.desc: "Open Telemetry Metrics configuration." otel_metrics.label: "Open Telemetry Metrics" +otel_traces.desc: "Open Telemetry Traces configuration." +otel_traces.label: "Open Telemetry Traces" + enable.desc: "Enable or disable Open Telemetry signal." enable.label: "Enable." @@ -41,4 +44,13 @@ otel_log_handler_level.desc: """The log level of the Open Telemetry log handler.""" otel_log_handler_level.label: "Log Level" +trace_filter.desc: "Open Telemetry Trace Filter configuration" +trace_filter.label: "Trace Filter" + +trace_all.desc: +"""If enabled, all published messages are traced, a new trace ID is generated if it can't be extracted from the message. +Otherwise, only messages published with trace context are traced. Disabled by default.""" +trace_all.label: "Trace All" + + } From 60da6427ca9fc6a5af234af1ac9067261389c250 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Mon, 20 Nov 2023 20:29:19 +0200 Subject: [PATCH 14/46] fix: move openetelmetry deps to emqx_openetelmetry app This is to fix elixir build, as mix can't find included opentelemetry hrl files. --- apps/emqx_opentelemetry/rebar.config | 14 ++++++-- .../src/emqx_opentelemetry.app.src | 7 ++++ mix.exs | 32 +------------------ rebar.config | 8 ----- 4 files changed, 19 insertions(+), 42 deletions(-) diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config index 7086a2f29..383ce1ba0 100644 --- a/apps/emqx_opentelemetry/rebar.config +++ b/apps/emqx_opentelemetry/rebar.config @@ -1,8 +1,16 @@ %% -*- mode: erlang -*- -{deps, [ - {emqx, {path, "../emqx"}} -]}. +{deps, + [{emqx, {path, "../emqx"}} + %% trace + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry"}} + %% log metrics + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_api_experimental"}} + %% export + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_exporter"}} + ]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 134ef2a44..13c4f73fa 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -10,6 +10,13 @@ %% otel metrics depend on emqx_mgmt_cache emqx_management ]}, + {included_applications, [ + opentelemetry, + opentelemetry_api, + opentelemetry_api_experimental, + opentelemetry_experimental, + opentelemetry_exporter + ]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/mix.exs b/mix.exs index 159b105a5..dc512399c 100644 --- a/mix.exs +++ b/mix.exs @@ -98,37 +98,7 @@ defmodule EMQXUmbrella.MixProject do # set by hackney (dependency) {:ssl_verify_fun, "1.1.7", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, - {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, - {:opentelemetry_api, - github: "emqx/opentelemetry-erlang", - sparse: "apps/opentelemetry_api", - tag: "v1.4.2-emqx", - override: true, - runtime: false}, - {:opentelemetry, - github: "emqx/opentelemetry-erlang", - sparse: "apps/opentelemetry", - tag: "v1.4.2-emqx", - override: true, - runtime: false}, - {:opentelemetry_api_experimental, - github: "emqx/opentelemetry-erlang", - sparse: "apps/opentelemetry_api_experimental", - tag: "v1.4.2-emqx", - override: true, - runtime: false}, - {:opentelemetry_experimental, - github: "emqx/opentelemetry-erlang", - sparse: "apps/opentelemetry_experimental", - tag: "v1.4.2-emqx", - override: true, - runtime: false}, - {:opentelemetry_exporter, - github: "emqx/opentelemetry-erlang", - sparse: "apps/opentelemetry_exporter", - tag: "v1.4.2-emqx", - override: true, - runtime: false} + {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index eec2d505a..2207f426d 100644 --- a/rebar.config +++ b/rebar.config @@ -84,14 +84,6 @@ %% in conflict by erlavro and rocketmq , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} - %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry"}} - %% log metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_api_experimental"}} - %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.2-emqx"}, "apps/opentelemetry_exporter"}} , {ssl_verify_fun, "1.1.7"} ]}. From 8d3f98eff22589c0eedbdb3542069b5cfbd2efac Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 21 Nov 2023 20:00:10 +0200 Subject: [PATCH 15/46] fix(emqx_opentelemetry): fix dialyzer warnings --- apps/emqx/src/emqx_external_trace.erl | 6 +++--- apps/emqx_opentelemetry/rebar.config | 10 +++++----- apps/emqx_opentelemetry/src/emqx_otel_metrics.erl | 6 +++--- apps/emqx_opentelemetry/src/emqx_otel_trace.erl | 10 ++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/apps/emqx/src/emqx_external_trace.erl b/apps/emqx/src/emqx_external_trace.erl index 7f8823903..fb32b8248 100644 --- a/apps/emqx/src/emqx_external_trace.erl +++ b/apps/emqx/src/emqx_external_trace.erl @@ -39,12 +39,12 @@ -define(PROVIDER, {?MODULE, trace_provider}). --define(with_provider(IfRegisitered, IfNotRegisired), +-define(with_provider(IfRegistered, IfNotRegistered), case persistent_term:get(?PROVIDER, undefined) of undefined -> - IfNotRegisired; + IfNotRegistered; Provider -> - Provider:IfRegisitered + Provider:IfRegistered end ). diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config index 383ce1ba0..3f215ff40 100644 --- a/apps/emqx_opentelemetry/rebar.config +++ b/apps/emqx_opentelemetry/rebar.config @@ -3,13 +3,13 @@ {deps, [{emqx, {path, "../emqx"}} %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry"}} + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry"}} %% log metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.3-emqx"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_exporter"}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl index 9ca1c5deb..7d3c9fa69 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl @@ -102,12 +102,12 @@ cleanup() -> safe_stop_default_metrics() -> try - _ = opentelemetry_experimental:stop_default_metrics() + _ = opentelemetry_experimental:stop_default_metrics(), + ok catch %% noramal scenario, metrics supervisor is not started exit:{noproc, _} -> ok - end, - ok. + end. create_metric_views() -> Meter = opentelemetry_experimental:get_meter(), diff --git a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl index 0c78f0abd..5220f1e8d 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl @@ -37,7 +37,6 @@ -include_lib("opentelemetry_api/include/otel_tracer.hrl"). -define(EMQX_OTEL_CTX, otel_ctx). -%% NOTE: it's possible to use trace_flags to set is_sampled flag -define(IS_ENABLED, emqx_enable). -define(USER_PROPERTY, 'User-Property'). @@ -115,13 +114,11 @@ trace_process_publish(Packet, Channel, ProcessFun) -> Ctx = otel_tracer:set_current_span(RootCtx1, SpanCtx), %% put ctx to packet, so it can be further propagated Packet1 = put_ctx_to_packet(Ctx, Packet), - %% TODO: consider getting rid of propagating Ctx through process dict as it's anyway seems to have - %% very limited usage - otel_ctx:attach(Ctx), + _ = otel_ctx:attach(Ctx), try ProcessFun(Packet1, Channel) after - ?end_span(), + _ = ?end_span(), clear() end end. @@ -261,7 +258,8 @@ clear() -> safe_stop_default_tracer() -> try - _ = opentelemetry:stop_default_tracer_provider() + _ = opentelemetry:stop_default_tracer_provider(), + ok catch %% noramal scenario, opentelemetry supervisor is not started exit:{noproc, _} -> ok From 2a3f6b749cd1af3d1751d1880c37734fe6dd3c09 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 23 Nov 2023 13:35:21 +0200 Subject: [PATCH 16/46] fix(emqx_opentelemetry): avoid using `application:ensure_all_started/1` for better deadlock safety --- apps/emqx_machine/priv/reboot_lists.eterm | 7 +--- apps/emqx_machine/src/emqx_machine.erl | 1 + apps/emqx_machine/src/emqx_machine_boot.erl | 4 +-- apps/emqx_opentelemetry/rebar.config | 12 +++---- .../src/emqx_opentelemetry.app.src | 10 +++--- apps/emqx_opentelemetry/src/emqx_otel_app.erl | 13 +++++-- .../src/emqx_otel_config.erl | 36 ------------------- .../src/emqx_otel_metrics.erl | 4 --- .../src/emqx_otel_trace.erl | 1 - 9 files changed, 23 insertions(+), 65 deletions(-) diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 27f984f51..6dffd5e2d 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -25,12 +25,7 @@ redbug, xmerl, {hocon, load}, - telemetry, - {opentelemetry, load}, - {opentelemetry_api, load}, - {opentelemetry_experimental, load}, - {opentelemetry_api_experimental, load}, - {opentelemetry_exporter, load} + telemetry ], %% must always be of type `load' common_business_apps => diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 36635d50e..8dc385fb3 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -50,6 +50,7 @@ start() -> start_sysmon(), configure_shard_transports(), set_mnesia_extra_diagnostic_checks(), + emqx_otel_app:configure_otel_deps(), ekka:start(), ok. diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 7abac0862..afb195543 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -69,9 +69,7 @@ stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), ok = emqx_conf_app:unset_config_loaded(), - lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())), - %% Mute otel deps application. - ok = emqx_otel_app:stop_deps(). + lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). %% Those port apps are terminated after the main apps %% Don't need to stop when reboot. diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config index 3f215ff40..0ba73216c 100644 --- a/apps/emqx_opentelemetry/rebar.config +++ b/apps/emqx_opentelemetry/rebar.config @@ -3,13 +3,13 @@ {deps, [{emqx, {path, "../emqx"}} %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry"}} - %% log metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry"}} + %% logs, metrics + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.4-emqx"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_exporter"}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 13c4f73fa..695ba7ae9 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -8,14 +8,12 @@ stdlib, emqx, %% otel metrics depend on emqx_mgmt_cache - emqx_management - ]}, - {included_applications, [ + emqx_management, + opentelemetry_exporter, opentelemetry, - opentelemetry_api, - opentelemetry_api_experimental, opentelemetry_experimental, - opentelemetry_exporter + opentelemetry_api, + opentelemetry_api_experimental ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_opentelemetry/src/emqx_otel_app.erl b/apps/emqx_opentelemetry/src/emqx_otel_app.erl index f4b579fe5..8231e089f 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_app.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_app.erl @@ -19,7 +19,7 @@ -behaviour(application). -export([start/2, stop/1]). --export([stop_deps/0]). +-export([configure_otel_deps/0]). start(_StartType, _StartArgs) -> emqx_otel_config:add_handler(), @@ -33,5 +33,12 @@ stop(_State) -> _ = emqx_otel_config:remove_otel_log_handler(), ok. -stop_deps() -> - emqx_otel_config:stop_all_otel_apps(). +configure_otel_deps() -> + %% default tracer and metrics are started only on demand + ok = application:set_env( + [ + {opentelemetry, [{start_default_tracer, false}]}, + {opentelemetry_experimental, [{start_default_metrics, false}]} + ], + [{persistent, true}] + ). diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index d2cc81521..f9ad6ca31 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -27,7 +27,6 @@ -export([post_config_update/5]). -export([update/1]). -export([add_otel_log_handler/0, remove_otel_log_handler/0]). --export([stop_all_otel_apps/0]). -export([otel_exporter/1]). update(Config) -> @@ -59,7 +58,6 @@ post_config_update(?OPTL, _Req, New, Old, AppEnvs) -> MetricsRes = ensure_otel_metrics(New, Old), LogsRes = ensure_otel_logs(New, Old), TracesRes = ensure_otel_traces(New, Old), - _ = maybe_stop_all_otel_apps(New), case {MetricsRes, LogsRes, TracesRes} of {ok, ok, ok} -> ok; Other -> {error, Other} @@ -67,9 +65,6 @@ post_config_update(?OPTL, _Req, New, Old, AppEnvs) -> post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> ok. -stop_all_otel_apps() -> - stop_all_otel_apps(true). - add_otel_log_handler() -> ensure_otel_logs(emqx:get_config(?OPTL), #{}). @@ -104,7 +99,6 @@ ensure_otel_logs(#{logs := LogsConf}, #{logs := LogsConf}) -> ok; ensure_otel_logs(#{logs := #{enable := true} = LogsConf}, _OldConf) -> ok = remove_handler_if_present(?OTEL_LOG_HANDLER_ID), - ok = ensure_log_apps(), HandlerConf = tr_handler_conf(LogsConf), %% NOTE: should primary logger level be updated if it's higher than otel log level? logger:add_handler(?OTEL_LOG_HANDLER_ID, ?OTEL_LOG_HANDLER, HandlerConf); @@ -126,21 +120,6 @@ remove_handler_if_present(HandlerId) -> ok end. -ensure_log_apps() -> - {ok, _} = application:ensure_all_started(opentelemetry_exporter), - {ok, _} = application:ensure_all_started(opentelemetry_experimental), - ok. - -maybe_stop_all_otel_apps(#{ - metrics := #{enable := false}, - logs := #{enable := false}, - traces := #{enable := false} -}) -> - IsShutdown = false, - stop_all_otel_apps(IsShutdown); -maybe_stop_all_otel_apps(_) -> - ok. - tr_handler_conf(Conf) -> #{ level := Level, @@ -171,18 +150,3 @@ is_ssl(<<"https://", _/binary>> = _Endpoint) -> true; is_ssl(_Endpoint) -> false. - -stop_all_otel_apps(IsShutdown) -> - %% if traces were enabled, it's not safe to stop opentelemetry app, - %% as there could be not finsihed traces that would crash if spans ETS tables are deleted - _ = - case IsShutdown of - true -> - _ = application:stop(opentelemetry); - false -> - ok - end, - _ = application:stop(opentelemetry_experimental), - _ = application:stop(opentelemetry_experimental_api), - _ = application:stop(opentelemetry_exporter), - ok. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl index 7d3c9fa69..1d70a800a 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl @@ -74,10 +74,6 @@ setup(_Conf) -> ensure_apps(Conf) -> #{exporter := #{interval := ExporterInterval} = Exporter} = Conf, - {ok, _} = application:ensure_all_started(opentelemetry_exporter), - {ok, _} = application:ensure_all_started(opentelemetry), - {ok, _} = application:ensure_all_started(opentelemetry_experimental), - {ok, _} = application:ensure_all_started(opentelemetry_api_experimental), _ = opentelemetry_experimental:stop_default_metrics(), ok = application:set_env( diff --git a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl index 5220f1e8d..e6bfd4749 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl @@ -78,7 +78,6 @@ start(Conf) -> ], set_trace_all(TraceAll), ok = application:set_env([{opentelemetry, OtelEnv}]), - _ = application:ensure_all_started(opentelemetry), Res = assert_started(opentelemetry:start_default_tracer_provider()), case Res of ok -> From c8e69357cc3bfb7b1c292387e7b1d02d1d593378 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 28 Nov 2023 15:16:47 +0200 Subject: [PATCH 17/46] feat(emqx_opentelemetry): use one global exporter config for all otel signals --- apps/emqx_opentelemetry/src/emqx_otel_app.erl | 2 +- .../src/emqx_otel_config.erl | 35 +++-- .../src/emqx_otel_metrics.erl | 7 +- .../src/emqx_otel_schema.erl | 135 ++++++++---------- apps/emqx_opentelemetry/src/emqx_otel_sup.erl | 6 +- .../src/emqx_otel_trace.erl | 10 +- rel/i18n/emqx_otel_schema.hocon | 4 +- 7 files changed, 95 insertions(+), 104 deletions(-) diff --git a/apps/emqx_opentelemetry/src/emqx_otel_app.erl b/apps/emqx_opentelemetry/src/emqx_otel_app.erl index 8231e089f..014785c6d 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_app.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_app.erl @@ -24,7 +24,7 @@ start(_StartType, _StartArgs) -> emqx_otel_config:add_handler(), ok = emqx_otel_config:add_otel_log_handler(), - ok = emqx_otel_trace:ensure_traces(emqx:get_config([opentelemetry, traces])), + ok = emqx_otel_trace:ensure_traces(emqx:get_config([opentelemetry])), emqx_otel_sup:start_link(). stop(_State) -> diff --git a/apps/emqx_opentelemetry/src/emqx_otel_config.erl b/apps/emqx_opentelemetry/src/emqx_otel_config.erl index f9ad6ca31..0d2f9988b 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_config.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_config.erl @@ -85,30 +85,40 @@ otel_exporter(ExporterConf) -> %% Internal functions -ensure_otel_metrics(#{metrics := MetricsConf}, #{metrics := MetricsConf}) -> +ensure_otel_metrics( + #{metrics := MetricsConf, exporter := Exporter}, + #{metrics := MetricsConf, exporter := Exporter} +) -> ok; -ensure_otel_metrics(#{metrics := #{enable := true} = MetricsConf}, _Old) -> +ensure_otel_metrics(#{metrics := #{enable := true}} = Conf, _Old) -> _ = emqx_otel_metrics:stop_otel(), - emqx_otel_metrics:start_otel(MetricsConf); + emqx_otel_metrics:start_otel(Conf); ensure_otel_metrics(#{metrics := #{enable := false}}, _Old) -> emqx_otel_metrics:stop_otel(); ensure_otel_metrics(_, _) -> ok. -ensure_otel_logs(#{logs := LogsConf}, #{logs := LogsConf}) -> +ensure_otel_logs( + #{logs := LogsConf, exporter := Exporter}, + #{logs := LogsConf, exporter := Exporter} +) -> ok; -ensure_otel_logs(#{logs := #{enable := true} = LogsConf}, _OldConf) -> +ensure_otel_logs(#{logs := #{enable := true}} = Conf, _OldConf) -> ok = remove_handler_if_present(?OTEL_LOG_HANDLER_ID), - HandlerConf = tr_handler_conf(LogsConf), + HandlerConf = tr_handler_conf(Conf), %% NOTE: should primary logger level be updated if it's higher than otel log level? logger:add_handler(?OTEL_LOG_HANDLER_ID, ?OTEL_LOG_HANDLER, HandlerConf); ensure_otel_logs(#{logs := #{enable := false}}, _OldConf) -> remove_handler_if_present(?OTEL_LOG_HANDLER_ID). -ensure_otel_traces(#{traces := TracesConf}, #{traces := TracesConf}) -> +ensure_otel_traces( + #{traces := TracesConf, exporter := Exporter}, + #{traces := TracesConf, exporter := Exporter} +) -> ok; -ensure_otel_traces(#{traces := #{enable := true} = TracesConf}, _OldConf) -> - emqx_otel_trace:start(TracesConf); +ensure_otel_traces(#{traces := #{enable := true}} = Conf, _OldConf) -> + _ = emqx_otel_trace:stop(), + emqx_otel_trace:start(Conf); ensure_otel_traces(#{traces := #{enable := false}}, _OldConf) -> emqx_otel_trace:stop(). @@ -120,14 +130,13 @@ remove_handler_if_present(HandlerId) -> ok end. -tr_handler_conf(Conf) -> +tr_handler_conf(#{logs := LogsConf, exporter := ExporterConf}) -> #{ level := Level, max_queue_size := MaxQueueSize, exporting_timeout := ExportingTimeout, - scheduled_delay := ScheduledDelay, - exporter := ExporterConf - } = Conf, + scheduled_delay := ScheduledDelay + } = LogsConf, #{ level => Level, config => #{ diff --git a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl index 1d70a800a..6e16e79d4 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_metrics.erl @@ -65,7 +65,7 @@ handle_info(_Msg, State) -> terminate(_Reason, _State) -> ok. -setup(Conf = #{enable := true}) -> +setup(Conf = #{metrics := #{enable := true}}) -> ensure_apps(Conf), create_metric_views(); setup(_Conf) -> @@ -73,7 +73,10 @@ setup(_Conf) -> ok. ensure_apps(Conf) -> - #{exporter := #{interval := ExporterInterval} = Exporter} = Conf, + #{ + exporter := Exporter, + metrics := #{interval := ExporterInterval} + } = Conf, _ = opentelemetry_experimental:stop_default_metrics(), ok = application:set_env( diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index 6359f88a5..bcd0b8dcf 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -30,15 +30,27 @@ upgrade_legacy_metrics(RawConf) -> case RawConf of #{<<"opentelemetry">> := Otel} -> - LegacyMetricsFields = [<<"enable">>, <<"exporter">>], - Otel1 = maps:without(LegacyMetricsFields, Otel), - Metrics = maps:with(LegacyMetricsFields, Otel), - case Metrics =:= #{} of - true -> - RawConf; - false -> - RawConf#{<<"opentelemetry">> => Otel1#{<<"metrics">> => Metrics}} - end; + Otel1 = + case maps:take(<<"enable">>, Otel) of + {MetricsEnable, OtelConf} -> + emqx_utils_maps:deep_put( + [<<"metrics">>, <<"enable">>], OtelConf, MetricsEnable + ); + error -> + Otel + end, + Otel2 = + case Otel1 of + #{<<"exporter">> := #{<<"interval">> := Interval} = Exporter} -> + emqx_utils_maps:deep_put( + [<<"metrics">>, <<"interval">>], + Otel1#{<<"exporter">> => maps:remove(<<"interval">>, Exporter)}, + Interval + ); + _ -> + Otel1 + end, + RawConf#{<<"opentelemetry">> => Otel2}; _ -> RawConf end. @@ -69,6 +81,13 @@ fields("opentelemetry") -> #{ desc => ?DESC(otel_traces) } + )}, + {exporter, + ?HOCON( + ?R_REF("otel_exporter"), + #{ + desc => ?DESC(otel_exporter) + } )} ]; fields("otel_metrics") -> @@ -82,10 +101,15 @@ fields("otel_metrics") -> desc => ?DESC(enable) } )}, - {exporter, + {interval, ?HOCON( - ?R_REF("otel_metrics_exporter"), - #{desc => ?DESC(exporter)} + emqx_schema:timeout_duration_ms(), + #{ + aliases => [scheduled_delay], + default => <<"10s">>, + desc => ?DESC(scheduled_delay), + importance => ?IMPORTANCE_HIDDEN + } )} ]; fields("otel_logs") -> @@ -134,14 +158,6 @@ fields("otel_logs") -> desc => ?DESC(scheduled_delay), importance => ?IMPORTANCE_HIDDEN } - )}, - {exporter, - ?HOCON( - ?R_REF("otel_logs_exporter"), - #{ - desc => ?DESC(exporter), - importance => ?IMPORTANCE_HIGH - } )} ]; fields("otel_traces") -> @@ -182,14 +198,6 @@ fields("otel_traces") -> importance => ?IMPORTANCE_HIDDEN } )}, - {exporter, - ?HOCON( - ?R_REF("otel_traces_exporter"), - #{ - desc => ?DESC(exporter), - importance => ?IMPORTANCE_HIGH - } - )}, {filter, ?HOCON( ?R_REF("trace_filter"), @@ -199,42 +207,7 @@ fields("otel_traces") -> } )} ]; -fields("otel_metrics_exporter") -> - exporter_fields(metrics); -fields("otel_logs_exporter") -> - exporter_fields(logs); -fields("ssl_opts") -> - Schema = emqx_schema:client_ssl_opts_schema(#{}), - lists:keydelete("enable", 1, Schema); -fields("otel_traces_exporter") -> - exporter_fields(traces); -fields("trace_filter") -> - %% More filters can be implemented in future, e.g. topic, clientid - [ - {trace_all, - ?HOCON( - boolean(), - #{ - default => false, - desc => ?DESC(trace_all), - importance => ?IMPORTANCE_MEDIUM - } - )} - ]. - -desc("opentelemetry") -> ?DESC(opentelemetry); -desc("exporter") -> ?DESC(exporter); -desc("otel_logs_exporter") -> ?DESC(exporter); -desc("otel_metrics_exporter") -> ?DESC(exporter); -desc("otel_traces_exporter") -> ?DESC(exporter); -desc("otel_logs") -> ?DESC(otel_logs); -desc("otel_metrics") -> ?DESC(otel_metrics); -desc("otel_traces") -> ?DESC(otel_traces); -desc("ssl_opts") -> ?DESC(exporter_ssl); -desc("trace_filter") -> ?DESC(trace_filter); -desc(_) -> undefined. - -exporter_fields(OtelSignal) -> +fields("otel_exporter") -> [ {endpoint, ?HOCON( @@ -263,21 +236,29 @@ exporter_fields(OtelSignal) -> importance => ?IMPORTANCE_LOW } )} - ] ++ exporter_extra_fields(OtelSignal). - -%% Let's keep it in exporter config for metrics, as it is different from -%% scheduled_delay_ms opt used for otel traces and logs -exporter_extra_fields(metrics) -> + ]; +fields("ssl_opts") -> + Schema = emqx_schema:client_ssl_opts_schema(#{}), + lists:keydelete("enable", 1, Schema); +fields("trace_filter") -> + %% More filters can be implemented in future, e.g. topic, clientid [ - {interval, + {trace_all, ?HOCON( - emqx_schema:timeout_duration_ms(), + boolean(), #{ - default => <<"10s">>, - required => true, - desc => ?DESC(scheduled_delay) + default => false, + desc => ?DESC(trace_all), + importance => ?IMPORTANCE_MEDIUM } )} - ]; -exporter_extra_fields(_OtelSignal) -> - []. + ]. + +desc("opentelemetry") -> ?DESC(opentelemetry); +desc("otel_exporter") -> ?DESC(otel_exporter); +desc("otel_logs") -> ?DESC(otel_logs); +desc("otel_metrics") -> ?DESC(otel_metrics); +desc("otel_traces") -> ?DESC(otel_traces); +desc("ssl_opts") -> ?DESC(exporter_ssl); +desc("trace_filter") -> ?DESC(trace_filter); +desc(_) -> undefined. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl index a823b2239..9d8165b21 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_sup.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_sup.erl @@ -41,8 +41,8 @@ init([]) -> period => 512 }, Children = - case emqx_conf:get([opentelemetry, metrics]) of - #{enable := false} -> []; - #{enable := true} = Conf -> [worker_spec(emqx_otel_metrics, Conf)] + case emqx_conf:get([opentelemetry]) of + #{metrics := #{enable := false}} -> []; + #{metrics := #{enable := true}} = Conf -> [worker_spec(emqx_otel_metrics, Conf)] end, {ok, {SupFlags, Children}}. diff --git a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl index e6bfd4749..061c3e983 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl @@ -55,26 +55,24 @@ toggle_registered(false = _Enable) -> ok. -spec ensure_traces(map()) -> ok | {error, term()}. -ensure_traces(#{enable := true} = Conf) -> +ensure_traces(#{traces := #{enable := true}} = Conf) -> start(Conf); ensure_traces(_Conf) -> ok. -spec start(map()) -> ok | {error, term()}. -start(Conf) -> - _ = safe_stop_default_tracer(), +start(#{traces := TracesConf, exporter := ExporterConf}) -> #{ - exporter := Exporter, max_queue_size := MaxQueueSize, exporting_timeout := ExportingTimeout, scheduled_delay := ScheduledDelay, filter := #{trace_all := TraceAll} - } = Conf, + } = TracesConf, OtelEnv = [ {bsp_scheduled_delay_ms, ScheduledDelay}, {bsp_exporting_timeout_ms, ExportingTimeout}, {bsp_max_queue_size, MaxQueueSize}, - {traces_exporter, emqx_otel_config:otel_exporter(Exporter)} + {traces_exporter, emqx_otel_config:otel_exporter(ExporterConf)} ], set_trace_all(TraceAll), ok = application:set_env([{opentelemetry, OtelEnv}]), diff --git a/rel/i18n/emqx_otel_schema.hocon b/rel/i18n/emqx_otel_schema.hocon index 0a41874b9..eca20b457 100644 --- a/rel/i18n/emqx_otel_schema.hocon +++ b/rel/i18n/emqx_otel_schema.hocon @@ -17,8 +17,8 @@ otel_traces.label: "Open Telemetry Traces" enable.desc: "Enable or disable Open Telemetry signal." enable.label: "Enable." -exporter.desc: "Open Telemetry Exporter" -exporter.label: "Exporter" +otel_exporter.desc: "Open Telemetry Exporter" +otel_exporter.label: "Exporter" max_queue_size.desc: """The maximum queue size. After the size is reached Open Telemetry signals are dropped.""" From 85441fda0f2f334cfc81e4d3653911418ce2cf26 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 29 Nov 2023 18:27:44 +0200 Subject: [PATCH 18/46] test(emqx_opentelemetry): add trace test suite --- .../docker-compose-otel.yaml | 69 +++ .ci/docker-compose-file/otel/.gitignore | 6 + .../otel/otel-collector-config-tls.yaml | 52 +++ .../otel/otel-collector-config.yaml | 51 +++ apps/emqx/src/emqx_external_trace.erl | 4 + apps/emqx_opentelemetry/docker-ct | 1 + apps/emqx_opentelemetry/src/emqx_otel_api.erl | 21 +- .../test/emqx_otel_api_SUITE.erl | 252 ++++++++++ .../test/emqx_otel_schema_SUITE.erl | 201 ++++++++ .../test/emqx_otel_trace_SUITE.erl | 431 ++++++++++++++++++ scripts/ct/run.sh | 3 + 11 files changed, 1078 insertions(+), 13 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-otel.yaml create mode 100644 .ci/docker-compose-file/otel/.gitignore create mode 100644 .ci/docker-compose-file/otel/otel-collector-config-tls.yaml create mode 100644 .ci/docker-compose-file/otel/otel-collector-config.yaml create mode 100644 apps/emqx_opentelemetry/docker-ct create mode 100644 apps/emqx_opentelemetry/test/emqx_otel_api_SUITE.erl create mode 100644 apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl create mode 100644 apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-otel.yaml b/.ci/docker-compose-file/docker-compose-otel.yaml new file mode 100644 index 000000000..a2d1bfae7 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-otel.yaml @@ -0,0 +1,69 @@ +version: '3.9' + +services: + jaeger-all-in-one: + image: jaegertracing/all-in-one:1.51.0 + container_name: jaeger.emqx.net + hostname: jaeger.emqx.net + networks: + - emqx_bridge + restart: always +# ports: +# - "16686:16686" + user: "${DOCKER_USER:-root}" + + # Collector + otel-collector: + image: otel/opentelemetry-collector:0.90.0 + container_name: otel-collector.emqx.net + hostname: otel-collector.emqx.net + networks: + - emqx_bridge + restart: always + command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"] + volumes: + - ./otel:/etc/ +# ports: +# - "1888:1888" # pprof extension +# - "8888:8888" # Prometheus metrics exposed by the collector +# - "8889:8889" # Prometheus exporter metrics +# - "13133:13133" # health_check extension +# - "4317:4317" # OTLP gRPC receiver +# - "4318:4318" # OTLP http receiver +# - "55679:55679" # zpages extension + depends_on: + - jaeger-all-in-one + user: "${DOCKER_USER:-root}" + + +# Collector + otel-collector-tls: + image: otel/opentelemetry-collector:0.90.0 + container_name: otel-collector-tls.emqx.net + hostname: otel-collector-tls.emqx.net + networks: + - emqx_bridge + restart: always + command: ["--config=/etc/otel-collector-config-tls.yaml", "${OTELCOL_ARGS}"] + volumes: + - ./otel:/etc/ + - ./certs:/etc/certs + # ports: + # - "14317:4317" # OTLP gRPC receiver + depends_on: + - jaeger-all-in-one + user: "${DOCKER_USER:-root}" + +#networks: +# emqx_bridge: +# driver: bridge +# name: emqx_bridge +# enable_ipv6: true +# ipam: +# driver: default +# config: +# - subnet: 172.100.239.0/24 +# gateway: 172.100.239.1 +# - subnet: 2001:3200:3200::/64 +# gateway: 2001:3200:3200::1 +# diff --git a/.ci/docker-compose-file/otel/.gitignore b/.ci/docker-compose-file/otel/.gitignore new file mode 100644 index 000000000..98dacbd74 --- /dev/null +++ b/.ci/docker-compose-file/otel/.gitignore @@ -0,0 +1,6 @@ +certs +hostname +hosts +otel-collector.json +otel-collector-tls.json +resolv.conf diff --git a/.ci/docker-compose-file/otel/otel-collector-config-tls.yaml b/.ci/docker-compose-file/otel/otel-collector-config-tls.yaml new file mode 100644 index 000000000..9163fc724 --- /dev/null +++ b/.ci/docker-compose-file/otel/otel-collector-config-tls.yaml @@ -0,0 +1,52 @@ +receivers: + otlp: + protocols: + grpc: + tls: + ca_file: /etc/certs/ca.crt + cert_file: /etc/certs/server.crt + key_file: /etc/certs/server.key + http: + tls: + ca_file: /etc/certs/ca.crt + cert_file: /etc/certs/server.crt + key_file: /etc/certs/server.key + +exporters: + logging: + verbosity: detailed + otlp: + endpoint: jaeger.emqx.net:4317 + tls: + insecure: true + debug: + verbosity: detailed + file: + path: /etc/otel-collector-tls.json + + +processors: + batch: + # send data immediately + timeout: 0 + +extensions: + health_check: + zpages: + endpoint: :55679 + +service: + extensions: [zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging] + logs: + receivers: [otlp] + processors: [batch] + exporters: [logging, file] diff --git a/.ci/docker-compose-file/otel/otel-collector-config.yaml b/.ci/docker-compose-file/otel/otel-collector-config.yaml new file mode 100644 index 000000000..6d6650139 --- /dev/null +++ b/.ci/docker-compose-file/otel/otel-collector-config.yaml @@ -0,0 +1,51 @@ +receivers: + otlp: + protocols: + grpc: + tls: +# ca_file: /etc/ca.pem +# cert_file: /etc/server.pem +# key_file: /etc/server.key + http: + tls: +# ca_file: /etc/ca.pem +# cert_file: /etc/server.pem +# key_file: /etc/server.key + +exporters: + logging: + verbosity: detailed + otlp: + endpoint: jaeger.emqx.net:4317 + tls: + insecure: true + debug: + verbosity: detailed + file: + path: /etc/otel-collector.json + +processors: + batch: + # send data immediately + timeout: 0 + +extensions: + health_check: + zpages: + endpoint: :55679 + +service: + extensions: [zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging] + logs: + receivers: [otlp] + processors: [batch] + exporters: [logging, file] diff --git a/apps/emqx/src/emqx_external_trace.erl b/apps/emqx/src/emqx_external_trace.erl index fb32b8248..b03e7139a 100644 --- a/apps/emqx/src/emqx_external_trace.erl +++ b/apps/emqx/src/emqx_external_trace.erl @@ -28,6 +28,7 @@ -callback event(EventName :: term(), Attributes :: term()) -> ok. -export([ + provider/0, register_provider/1, unregister_provider/1, trace_process_publish/3, @@ -71,6 +72,9 @@ unregister_provider(Module) -> {error, not_registered} end. +-spec provider() -> module() | undefined. +provider() -> + persistent_term:get(?PROVIDER, undefined). %%-------------------------------------------------------------------- %% trace API %%-------------------------------------------------------------------- diff --git a/apps/emqx_opentelemetry/docker-ct b/apps/emqx_opentelemetry/docker-ct new file mode 100644 index 000000000..8f7569d06 --- /dev/null +++ b/apps/emqx_opentelemetry/docker-ct @@ -0,0 +1 @@ +otel diff --git a/apps/emqx_opentelemetry/src/emqx_otel_api.erl b/apps/emqx_opentelemetry/src/emqx_otel_api.erl index d8c76ebcf..f14bdb00b 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_api.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_api.erl @@ -103,24 +103,19 @@ otel_config_schema() -> otel_config_example() -> #{ + exporter => #{ + endpoint => "http://localhost:4317", + ssl_options => #{} + }, logs => #{ enable => true, - exporter => #{ - endpoint => "http://localhost:4317", - ssl_options => #{ - enable => false - } - }, level => warning }, metrics => #{ + enable => true + }, + traces => #{ enable => true, - exporter => #{ - endpoint => "http://localhost:4317", - interval => "10s", - ssl_options => #{ - enable => false - } - } + filter => #{trace_all => false} } }. diff --git a/apps/emqx_opentelemetry/test/emqx_otel_api_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_api_SUITE.erl new file mode 100644 index 000000000..f829ca640 --- /dev/null +++ b/apps/emqx_opentelemetry/test/emqx_otel_api_SUITE.erl @@ -0,0 +1,252 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(OTEL_API_PATH, emqx_mgmt_api_test_util:api_path(["opentelemetry"])). +-define(CONF_PATH, [opentelemetry]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + %% This is called by emqx_machine in EMQX release + emqx_otel_app:configure_otel_deps(), + Apps = emqx_cth_suite:start( + [ + emqx_conf, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, + emqx_opentelemetry + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + Auth = auth_header(), + [{suite_apps, Apps}, {auth, Auth} | Config]. + +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)), + emqx_config:delete_override_conf_files(), + ok. + +init_per_testcase(_TC, Config) -> + emqx_conf:update( + ?CONF_PATH, + #{ + <<"traces">> => #{<<"enable">> => false}, + <<"metrics">> => #{<<"enable">> => false}, + <<"logs">> => #{<<"enable">> => false} + }, + #{} + ), + Config. + +end_per_testcase(_TC, _Config) -> + ok. + +auth_header() -> + {ok, API} = emqx_common_test_http:create_default_app(), + emqx_common_test_http:auth_header(API). + +t_get(Config) -> + Auth = ?config(auth, Config), + Path = ?OTEL_API_PATH, + {ok, Resp} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), + ?assertMatch( + #{ + <<"traces">> := #{<<"enable">> := false}, + <<"metrics">> := #{<<"enable">> := false}, + <<"logs">> := #{<<"enable">> := false} + }, + emqx_utils_json:decode(Resp) + ). + +t_put_enable_disable(Config) -> + Auth = ?config(auth, Config), + Path = ?OTEL_API_PATH, + EnableAllReq = #{ + <<"traces">> => #{<<"enable">> => true}, + <<"metrics">> => #{<<"enable">> => true}, + <<"logs">> => #{<<"enable">> => true} + }, + ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, EnableAllReq)), + ?assertMatch( + #{ + traces := #{enable := true}, + metrics := #{enable := true}, + logs := #{enable := true} + }, + emqx:get_config(?CONF_PATH) + ), + + DisableAllReq = #{ + <<"traces">> => #{<<"enable">> => false}, + <<"metrics">> => #{<<"enable">> => false}, + <<"logs">> => #{<<"enable">> => false} + }, + ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, DisableAllReq)), + ?assertMatch( + #{ + traces := #{enable := false}, + metrics := #{enable := false}, + logs := #{enable := false} + }, + emqx:get_config(?CONF_PATH) + ). + +t_put_invalid(Config) -> + Auth = ?config(auth, Config), + Path = ?OTEL_API_PATH, + + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"endpoint">> => <<>>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"endpoint">> => <<"unknown://somehost.org">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"endpoint">> => <<"https://somehost.org:99999">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"endpoint">> => <<"https://somehost.org:99999">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"unknown_field">> => <<"foo">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"protocol">> => <<"unknown">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"traces">> => #{<<"filter">> => #{<<"unknown_filter">> => <<"foo">>}} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"logs">> => #{<<"level">> => <<"foo">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"metrics">> => #{<<"interval">> => <<"foo">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"logs">> => #{<<"unknown_field">> => <<"foo">>} + }) + ), + ?assertMatch( + {error, {_, 400, _}}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{<<"unknown_field">> => <<"foo">>}) + ). + +t_put_valid(Config) -> + Auth = ?config(auth, Config), + Path = ?OTEL_API_PATH, + + ?assertMatch( + {ok, _}, + emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{ + <<"exporter">> => #{<<"endpoint">> => <<"nohost.com">>} + }) + ), + ?assertEqual(<<"http://nohost.com/">>, emqx:get_config(?CONF_PATH ++ [exporter, endpoint])), + + ?assertMatch( + {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{<<"exporter">> => #{}}) + ), + ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{})), + ?assertMatch( + {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{<<"traces">> => #{}}) + ), + ?assertMatch( + {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{<<"logs">> => #{}}) + ), + ?assertMatch( + {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, #{<<"metrics">> => #{}}) + ), + ?assertMatch( + {ok, _}, + emqx_mgmt_api_test_util:request_api( + put, + Path, + "", + Auth, + #{<<"exporter">> => #{}, <<"traces">> => #{}, <<"logs">> => #{}, <<"metrics">> => #{}} + ) + ), + ?assertMatch( + {ok, _}, + emqx_mgmt_api_test_util:request_api( + put, + Path, + "", + Auth, + #{ + <<"exporter">> => #{ + <<"endpoint">> => <<"https://localhost:4317">>, <<"protocol">> => <<"grpc">> + }, + <<"traces">> => #{ + <<"enable">> => true, + <<"max_queue_size">> => 10, + <<"exporting_timeout">> => <<"10s">>, + <<"scheduled_delay">> => <<"20s">>, + <<"filter">> => #{<<"trace_all">> => true} + }, + <<"logs">> => #{ + <<"level">> => <<"warning">>, + <<"max_queue_size">> => 100, + <<"exporting_timeout">> => <<"10s">>, + <<"scheduled_delay">> => <<"1s">> + }, + <<"metrics">> => #{ + %% alias for "interval" + <<"scheduled_delay">> => <<"15321ms">> + } + } + ), + %% alias check + ?assertEqual(15_321, emqx:get_config(?CONF_PATH ++ [metrics, interval])) + ). diff --git a/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl new file mode 100644 index 000000000..f5682dcad --- /dev/null +++ b/apps/emqx_opentelemetry/test/emqx_otel_schema_SUITE.erl @@ -0,0 +1,201 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_otel_schema_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% Backward compatibility suite for `upgrade_raw_conf/1`, +%% expected callback is `emqx_otel_schema:upgrade_legacy_metrics/1` + +-define(OLD_CONF_ENABLED, << + "\n" + "opentelemetry\n" + "{\n" + " enable = true\n" + "}\n" +>>). + +-define(OLD_CONF_DISABLED, << + "\n" + "opentelemetry\n" + "{\n" + " enable = false\n" + "}\n" +>>). + +-define(OLD_CONF_ENABLED_EXPORTER, << + "\n" + "opentelemetry\n" + "{\n" + " enable = true\n" + " exporter {endpoint = \"http://127.0.0.1:4317/\", interval = 5s}\n" + "}\n" +>>). + +-define(OLD_CONF_DISABLED_EXPORTER, << + "\n" + "opentelemetry\n" + "{\n" + " enable = false\n" + " exporter {endpoint = \"http://127.0.0.1:4317/\", interval = 5s}\n" + "}\n" +>>). + +-define(OLD_CONF_EXPORTER, << + "\n" + "opentelemetry\n" + "{\n" + " exporter {endpoint = \"http://127.0.0.1:4317/\", interval = 5s}\n" + "}\n" +>>). + +-define(OLD_CONF_EXPORTER_PARTIAL, << + "\n" + "opentelemetry\n" + "{\n" + " exporter {endpoint = \"http://127.0.0.1:4317/\"}\n" + "}\n" +>>). + +-define(OLD_CONF_EXPORTER_PARTIAL1, << + "\n" + "opentelemetry\n" + "{\n" + " exporter {interval = 3s}\n" + "}\n" +>>). + +-define(TESTS_CONF, #{ + t_old_conf_enabled => ?OLD_CONF_ENABLED, + t_old_conf_disabled => ?OLD_CONF_DISABLED, + t_old_conf_enabled_exporter => ?OLD_CONF_ENABLED_EXPORTER, + t_old_conf_disabled_exporter => ?OLD_CONF_DISABLED_EXPORTER, + t_old_conf_exporter => ?OLD_CONF_EXPORTER, + t_old_conf_exporter_partial => ?OLD_CONF_EXPORTER_PARTIAL, + t_old_conf_exporter_partial1 => ?OLD_CONF_EXPORTER_PARTIAL1 +}). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(TC, Config) -> + Apps = start_apps(TC, Config, maps:get(TC, ?TESTS_CONF)), + [{suite_apps, Apps} | Config]. + +end_per_testcase(_TC, Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)), + emqx_config:delete_override_conf_files(), + ok. + +start_apps(TC, Config, OtelConf) -> + emqx_cth_suite:start( + [ + {emqx_conf, OtelConf}, + emqx_management, + emqx_opentelemetry + ], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ). + +t_old_conf_enabled(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{metrics := #{enable := true, interval := _}, exporter := #{endpoint := _}}, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_disabled(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{metrics := #{enable := false, interval := _}, exporter := #{endpoint := _}}, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_enabled_exporter(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{ + metrics := #{enable := true, interval := 5000}, + exporter := #{endpoint := <<"http://127.0.0.1:4317/">>} + }, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_disabled_exporter(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{ + metrics := #{enable := false, interval := 5000}, + exporter := #{endpoint := <<"http://127.0.0.1:4317/">>} + }, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_exporter(_Config) -> + io:format(user, "TC running: ~p~n", [?FUNCTION_NAME]), + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{ + metrics := #{enable := false, interval := 5000}, + exporter := #{endpoint := <<"http://127.0.0.1:4317/">>} + }, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_exporter_partial(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{ + metrics := #{enable := false, interval := _}, + exporter := #{endpoint := <<"http://127.0.0.1:4317/">>} + }, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). + +t_old_conf_exporter_partial1(_Config) -> + OtelConf = emqx:get_config([opentelemetry]), + ?assertMatch( + #{ + metrics := #{enable := false, interval := 3000}, + exporter := #{endpoint := _} + }, + OtelConf + ), + ?assertNot(erlang:is_map_key(enable, OtelConf)), + ?assertNot(erlang:is_map_key(interval, maps:get(exporter, OtelConf))). diff --git a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl new file mode 100644 index 000000000..88917d7e3 --- /dev/null +++ b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl @@ -0,0 +1,431 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_otel_trace_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(OTEL_SERVICE_NAME, "emqx"). +-define(CONF_PATH, [opentelemetry]). + +%% How to run it locally: +%% 1. Uncomment networks in .ci/docker-compose-file/docker-compose-otel.yaml, +%% Uncomment OTLP gRPC ports mappings for otel-collector and otel-collector-tls services. +%% Uncomment jaeger-all-in-one prots maooing. +%% 2. Start deps services: +%% DOCKER_USER="$(id -u)" docker-compose -f .ci/docker-compose-file/docker-compose-otel.yaml up +%% 3. Run tests with special env variables: +%% PROFILE=emqx JAEGER_URL="http://localhost:16686" \ +%% OTEL_COLLECTOR_URL="http://localhost:4317" OTEL_COLLECTOR_TLS_URL="https://localhost:14317" \ +%% make "apps/emqx_opentelemetry-ct" +%% Or run only this suite: +%% PROFILE=emqx JAEGER_URL="http://localhost:16686" \ +%% OTEL_COLLECTOR_URL="http://localhost:4317" OTEL_COLLECTOR_TLS_URL="https://localhost:14317" \ +%% ./rebar3 ct -v --readable=true --name 'test@127.0.0.1' \ +%% --suite apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl + +all() -> + [ + {group, tcp}, + {group, tls} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {tcp, TCs}, + {tls, TCs} + ]. + +init_per_suite(Config) -> + %% This is called by emqx_machine in EMQX release + emqx_otel_app:configure_otel_deps(), + %% No release name during the test case, we need a reliable service name to query Jaeger + os:putenv("OTEL_SERVICE_NAME", ?OTEL_SERVICE_NAME), + JaegerURL = os:getenv("JAEGER_URL", "http://jaeger.emqx.net:16686"), + [{jaeger_url, JaegerURL} | Config]. + +end_per_suite(_) -> + os:unsetenv("OTEL_SERVICE_NAME"), + ok. + +init_per_group(tcp = Group, Config) -> + OtelCollectorURL = os:getenv("OTEL_COLLECTOR_URL", "http://otel-collector.emqx.net:4317"), + [ + {otel_collector_url, OtelCollectorURL}, + {logs_exporter_file_path, logs_exporter_file_path(Group, Config)} + | Config + ]; +init_per_group(tls = Group, Config) -> + OtelCollectorURL = os:getenv( + "OTEL_COLLECTOR_TLS_URL", "https://otel-collector-tls.emqx.net:4317" + ), + [ + {otel_collector_url, OtelCollectorURL}, + {logs_exporter_file_path, logs_exporter_file_path(Group, Config)} + | Config + ]. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(t_distributed_trace = TC, Config) -> + Cluster = cluster(TC, Config), + [{cluster, Cluster} | Config]; +init_per_testcase(TC, Config) -> + Apps = emqx_cth_suite:start(apps_spec(), #{work_dir => emqx_cth_suite:work_dir(TC, Config)}), + [{suite_apps, Apps} | Config]. + +end_per_testcase(t_distributed_trace = _TC, Config) -> + emqx_cth_cluster:stop(?config(cluster, Config)), + emqx_config:delete_override_conf_files(), + ok; +end_per_testcase(_TC, Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)), + emqx_config:delete_override_conf_files(), + ok. + +t_trace(Config) -> + MqttHostPort = mqtt_host_port(), + + {ok, _} = emqx_conf:update(?CONF_PATH, enabled_trace_conf(Config), #{override_to => cluster}), + + Topic = <<"t/trace/test/", (atom_to_binary(?FUNCTION_NAME))/binary>>, + TopicNoSubs = <<"t/trace/test/nosub/", (atom_to_binary(?FUNCTION_NAME))/binary>>, + + SubConn1 = connect(MqttHostPort, <<"sub1">>), + {ok, _, [0]} = emqtt:subscribe(SubConn1, Topic), + SubConn2 = connect(MqttHostPort, <<"sub2">>), + {ok, _, [0]} = emqtt:subscribe(SubConn2, Topic), + PubConn = connect(MqttHostPort, <<"pub">>), + + TraceParent = traceparent(true), + TraceParentNotSampled = traceparent(false), + ok = emqtt:publish(PubConn, Topic, props(TraceParent), <<"must be traced">>, []), + ok = emqtt:publish(PubConn, Topic, props(TraceParentNotSampled), <<"must not be traced">>, []), + + TraceParentNoSub = traceparent(true), + TraceParentNoSubNotSampled = traceparent(false), + ok = emqtt:publish(PubConn, TopicNoSubs, props(TraceParentNoSub), <<"must be traced">>, []), + ok = emqtt:publish( + PubConn, TopicNoSubs, props(TraceParentNoSubNotSampled), <<"must not be traced">>, [] + ), + + ?assertEqual( + ok, + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + {ok, #{<<"data">> := Traces}} = get_jaeger_traces(?config(jaeger_url, Config)), + [Trace] = filter_traces(trace_id(TraceParent), Traces), + [] = filter_traces(trace_id(TraceParentNotSampled), Traces), + [TraceNoSub] = filter_traces(trace_id(TraceParentNoSub), Traces), + [] = filter_traces(trace_id(TraceParentNoSubNotSampled), Traces), + + #{<<"spans">> := Spans, <<"processes">> := _} = Trace, + %% 2 sub spans and 1 publish process span + IsExpectedSpansLen = length(Spans) =:= 3, + + #{<<"spans">> := SpansNoSub, <<"processes">> := _} = TraceNoSub, + %% Only 1 publish process span + IsExpectedSpansLen andalso 1 =:= length(SpansNoSub) + end, + 10_000 + ) + ), + stop_conns([SubConn1, SubConn2, PubConn]). + +t_trace_disabled(_Config) -> + ?assertNot(emqx:get_config(?CONF_PATH ++ [traces, enable])), + %% Tracer must be actually disabled + ?assertEqual({otel_tracer_noop, []}, opentelemetry:get_tracer()), + ?assertEqual(undefined, emqx_external_trace:provider()), + + Topic = <<"t/trace/test", (atom_to_binary(?FUNCTION_NAME))/binary>>, + + SubConn = connect(mqtt_host_port(), <<"sub">>), + {ok, _, [0]} = emqtt:subscribe(SubConn, Topic), + PubConn = connect(mqtt_host_port(), <<"pub">>), + + TraceParent = traceparent(true), + emqtt:publish(PubConn, Topic, props(TraceParent), <<>>, []), + receive + {publish, #{topic := Topic, properties := Props}} -> + %% traceparent must be propagated by EMQX even if internal otel trace is disabled + #{'User-Property' := [{<<"traceparent">>, TrParent}]} = Props, + ?assertEqual(TraceParent, TrParent) + after 10_000 -> + ct:fail("published_message_not_received") + end, + + %% if otel trace is registered but is actually not running, EMQX must work fine + %% and the message must be delivered to the subscriber + ok = emqx_otel_trace:toggle_registered(true), + TraceParent1 = traceparent(true), + emqtt:publish(PubConn, Topic, props(TraceParent1), <<>>, []), + receive + {publish, #{topic := Topic, properties := Props1}} -> + #{'User-Property' := [{<<"traceparent">>, TrParent1}]} = Props1, + ?assertEqual(TraceParent1, TrParent1) + after 10_000 -> + ct:fail("published_message_not_received") + end, + stop_conns([SubConn, PubConn]). + +t_trace_all(Config) -> + OtelConf = enabled_trace_conf(Config), + OtelConf1 = emqx_utils_maps:deep_put([<<"traces">>, <<"filter">>], OtelConf, #{ + <<"trace_all">> => true + }), + {ok, _} = emqx_conf:update(?CONF_PATH, OtelConf1, #{override_to => cluster}), + + Topic = <<"t/trace/test", (atom_to_binary(?FUNCTION_NAME))/binary>>, + ClientId = <<"pub-", (integer_to_binary(erlang:system_time(nanosecond)))/binary>>, + PubConn = connect(mqtt_host_port(), ClientId), + emqtt:publish(PubConn, Topic, #{}, <<>>, []), + + ?assertEqual( + ok, + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + {ok, #{<<"data">> := Traces}} = get_jaeger_traces(?config(jaeger_url, Config)), + Res = lists:filter( + fun(#{<<"spans">> := Spans}) -> + case Spans of + %% Only one span is expected as there are no subscribers + [#{<<"tags">> := Tags}] -> + lists:any( + fun(#{<<"key">> := K, <<"value">> := Val}) -> + K =:= <<"messaging.client_id">> andalso Val =:= ClientId + end, + Tags + ); + _ -> + false + end + end, + Traces + ), + %% Expecting exactly 1 span + length(Res) =:= 1 + end, + 10_000 + ) + ), + stop_conns([PubConn]). + +t_distributed_trace(Config) -> + [Core1, Core2, Repl] = Cluster = ?config(cluster, Config), + {ok, _} = rpc:call( + Core1, + emqx_conf, + update, + [?CONF_PATH, enabled_trace_conf(Config), #{override_to => cluster}] + ), + Topic = <<"t/trace/test/", (atom_to_binary(?FUNCTION_NAME))/binary>>, + + SubConn1 = connect(mqtt_host_port(Core1), <<"sub1">>), + {ok, _, [0]} = emqtt:subscribe(SubConn1, Topic), + SubConn2 = connect(mqtt_host_port(Core2), <<"sub2">>), + {ok, _, [0]} = emqtt:subscribe(SubConn2, Topic), + SubConn3 = connect(mqtt_host_port(Repl), <<"sub3">>), + {ok, _, [0]} = emqtt:subscribe(SubConn3, Topic), + + PubConn = connect(mqtt_host_port(Repl), <<"pub">>), + + TraceParent = traceparent(true), + TraceParentNotSampled = traceparent(false), + + ok = emqtt:publish(PubConn, Topic, props(TraceParent), <<"must be traced">>, []), + ok = emqtt:publish(PubConn, Topic, props(TraceParentNotSampled), <<"must not be traced">>, []), + + ?assertEqual( + ok, + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + {ok, #{<<"data">> := Traces}} = get_jaeger_traces(?config(jaeger_url, Config)), + [Trace] = filter_traces(trace_id(TraceParent), Traces), + + [] = filter_traces(trace_id(TraceParentNotSampled), Traces), + + #{<<"spans">> := Spans, <<"processes">> := Procs} = Trace, + + %% 3 sub spans and 1 publish process span + 4 = length(Spans), + [_, _, _] = SendSpans = filter_spans(<<"send_published_message">>, Spans), + + IsAllNodesSpans = + lists:sort([atom_to_binary(N) || N <- Cluster]) =:= + lists:sort([span_node(S, Procs) || S <- SendSpans]), + + [PubSpan] = filter_spans(<<"process_message">>, Spans), + atom_to_binary(Repl) =:= span_node(PubSpan, Procs) andalso IsAllNodesSpans + end, + 10_000 + ) + ), + stop_conns([SubConn1, SubConn2, SubConn3, PubConn]). + +%% Keeping this test in this SUITE as there is no separate module for logs +t_log(Config) -> + Level = emqx_logger:get_primary_log_level(), + LogsConf = #{ + <<"logs">> => #{ + <<"enable">> => true, + <<"level">> => atom_to_binary(Level), + <<"scheduled_delay">> => <<"20ms">> + }, + <<"exporter">> => exporter_conf(Config) + }, + {ok, _} = emqx_conf:update(?CONF_PATH, LogsConf, #{override_to => cluster}), + + %% Ids are only needed for matching logs in the file exported by otel-collector + Id = integer_to_binary(otel_id_generator:generate_trace_id()), + ?SLOG(Level, #{msg => "otel_test_log_message", id => Id}), + Id1 = integer_to_binary(otel_id_generator:generate_trace_id()), + logger:Level("Ordinary log message, id: ~p", [Id1]), + + ?assertEqual( + ok, + emqx_common_test_helpers:wait_for( + ?FUNCTION_NAME, + ?LINE, + fun() -> + {ok, Logs} = file:read_file(?config(logs_exporter_file_path, Config)), + binary:match(Logs, Id) =/= nomatch andalso binary:match(Logs, Id1) =/= nomatch + end, + 10_000 + ) + ). + +logs_exporter_file_path(Group, Config) -> + filename:join([project_dir(Config), logs_exporter_filename(Group)]). + +project_dir(Config) -> + filename:join( + lists:takewhile( + fun(PathPart) -> PathPart =/= "_build" end, + filename:split(?config(priv_dir, Config)) + ) + ). + +logs_exporter_filename(tcp) -> + ".ci/docker-compose-file/otel/otel-collector.json"; +logs_exporter_filename(tls) -> + ".ci/docker-compose-file/otel/otel-collector-tls.json". + +enabled_trace_conf(TcConfig) -> + #{ + <<"traces">> => #{ + <<"enable">> => true, + <<"scheduled_delay">> => <<"50ms">> + }, + <<"exporter">> => exporter_conf(TcConfig) + }. + +exporter_conf(TcConfig) -> + #{<<"endpoint">> => ?config(otel_collector_url, TcConfig)}. + +span_node(#{<<"processID">> := ProcId}, Procs) -> + #{ProcId := #{<<"tags">> := ProcTags}} = Procs, + [#{<<"value">> := Node}] = lists:filter( + fun(#{<<"key">> := K}) -> + K =:= <<"service.instance.id">> + end, + ProcTags + ), + Node. + +trace_id(<<"00-", TraceId:32/binary, _/binary>>) -> + TraceId. + +filter_traces(TraceId, Traces) -> + lists:filter(fun(#{<<"traceID">> := TrId}) -> TrId =:= TraceId end, Traces). + +filter_spans(OpName, Spans) -> + lists:filter(fun(#{<<"operationName">> := Name}) -> Name =:= OpName end, Spans). + +get_jaeger_traces(JagerBaseURL) -> + case httpc:request(JagerBaseURL ++ "/api/traces?service=" ++ ?OTEL_SERVICE_NAME) of + {ok, {{_, 200, _}, _, RespBpdy}} -> + {ok, emqx_utils_json:decode(RespBpdy)}; + Err -> + ct:pal("Jager error: ~p", Err), + Err + end. + +stop_conns(Conns) -> + lists:foreach(fun emqtt:stop/1, Conns). + +props(TraceParent) -> + #{'User-Property' => [{<<"traceparent">>, TraceParent}]}. + +traceparent(IsSampled) -> + TraceId = otel_id_generator:generate_trace_id(), + SpanId = otel_id_generator:generate_span_id(), + {ok, TraceIdHexStr} = otel_utils:format_binary_string("~32.16.0b", [TraceId]), + {ok, SpanIdHexStr} = otel_utils:format_binary_string("~16.16.0b", [SpanId]), + TraceFlags = + case IsSampled of + true -> <<"01">>; + false -> <<"00">> + end, + <<"00-", TraceIdHexStr/binary, "-", SpanIdHexStr/binary, "-", TraceFlags/binary>>. + +connect({Host, Port}, ClientId) -> + {ok, ConnPid} = emqtt:start_link([ + {proto_ver, v5}, + {host, Host}, + {port, Port}, + {clientid, ClientId} + ]), + {ok, _} = emqtt:connect(ConnPid), + ConnPid. + +mqtt_host_port() -> + emqx:get_config([listeners, tcp, default, bind]). + +mqtt_host_port(Node) -> + rpc:call(Node, emqx, get_config, [[listeners, tcp, default, bind]]). + +cluster(TC, Config) -> + Nodes = emqx_cth_cluster:start( + [ + {otel_trace_core1, #{role => core, apps => apps_spec()}}, + {otel_trace_core2, #{role => core, apps => apps_spec()}}, + {otel_trace_replicant, #{role => replicant, apps => apps_spec()}} + ], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + Nodes. + +apps_spec() -> + [ + emqx, + emqx_conf, + emqx_management, + emqx_opentelemetry + ]. diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4de3b4d7c..7959581a9 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -243,6 +243,9 @@ for dep in ${CT_DEPS}; do ldap) FILES+=( '.ci/docker-compose-file/docker-compose-ldap.yaml' ) ;; + otel) + FILES+=( '.ci/docker-compose-file/docker-compose-otel.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From 195a23ae27198dbbf02e97adfadc0ebec6d38bca Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 5 Dec 2023 17:44:58 +0200 Subject: [PATCH 19/46] chore(emqx_opentelemetry): bump opentelemetry to v1.4.6-emqx --- apps/emqx_opentelemetry/rebar.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_opentelemetry/rebar.config b/apps/emqx_opentelemetry/rebar.config index 0ba73216c..a14c5922e 100644 --- a/apps/emqx_opentelemetry/rebar.config +++ b/apps/emqx_opentelemetry/rebar.config @@ -3,13 +3,13 @@ {deps, [{emqx, {path, "../emqx"}} %% trace - , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_api"}} - , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry"}} + , {opentelemetry_api, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_api"}} + , {opentelemetry, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry"}} %% logs, metrics - , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_experimental"}} - , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_api_experimental"}} + , {opentelemetry_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_experimental"}} + , {opentelemetry_api_experimental, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_api_experimental"}} %% export - , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.5-emqx"}, "apps/opentelemetry_exporter"}} + , {opentelemetry_exporter, {git_subdir, "https://github.com/emqx/opentelemetry-erlang", {tag, "v1.4.6-emqx"}, "apps/opentelemetry_exporter"}} ]}. {edoc_opts, [{preprocess, true}]}. From f30c97f1900ad2059fd615da492dcda3cf694f89 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 5 Dec 2023 13:26:02 -0300 Subject: [PATCH 20/46] chore: bump app vsn --- apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index 10dd6efef..ba43cabc8 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, From 938508b270c40bef20cf3f41e7344abfa9ceac73 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Tue, 5 Dec 2023 19:01:08 +0200 Subject: [PATCH 21/46] refactor(emqx_opentelemetry): pass only channel info to trace functions --- apps/emqx/src/emqx_channel.erl | 13 ++++++++-- apps/emqx/src/emqx_external_trace.erl | 24 ++++++++++------- .../src/emqx_otel_trace.erl | 26 +++++++++---------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 686d524dc..e52ac50b0 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -399,7 +399,12 @@ handle_in(?PACKET(_), Channel = #channel{conn_state = ConnState}) when handle_in(Packet = ?PUBLISH_PACKET(_QoS), Channel) -> case emqx_packet:check(Packet) of ok -> - emqx_external_trace:trace_process_publish(Packet, Channel, fun process_publish/2); + emqx_external_trace:trace_process_publish( + Packet, + %% More info can be added in future, but for now only clientid is used + trace_info(Channel), + fun(PacketWithTrace) -> process_publish(PacketWithTrace, Channel) end + ); {error, ReasonCode} -> handle_out(disconnect, ReasonCode, Channel) end; @@ -924,7 +929,7 @@ handle_deliver( NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session), {ok, Channel#channel{session = NSession}}; handle_deliver(Delivers, Channel) -> - Delivers1 = emqx_external_trace:start_trace_send(Delivers, Channel), + Delivers1 = emqx_external_trace:start_trace_send(Delivers, trace_info(Channel)), do_handle_deliver(Delivers1, Channel). do_handle_deliver( @@ -1435,6 +1440,10 @@ overload_protection(_, #channel{clientinfo = #{zone := Zone}}) -> emqx_olp:backoff(Zone), ok. +trace_info(Channel) -> + %% More info can be added in future, but for now only clientid is used + maps:from_list(info([clientid], Channel)). + %%-------------------------------------------------------------------- %% Enrich MQTT Connect Info diff --git a/apps/emqx/src/emqx_external_trace.erl b/apps/emqx/src/emqx_external_trace.erl index b03e7139a..1a7df93d0 100644 --- a/apps/emqx/src/emqx_external_trace.erl +++ b/apps/emqx/src/emqx_external_trace.erl @@ -15,18 +15,20 @@ %%-------------------------------------------------------------------- -module(emqx_external_trace). --callback trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when +-callback trace_process_publish(Packet, ChannelInfo, fun((Packet) -> Res)) -> Res when Packet :: emqx_types:packet(), - Channel :: emqx_channel:channel(), + ChannelInfo :: channel_info(), Res :: term(). --callback start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> +-callback start_trace_send(list(emqx_types:deliver()), channel_info()) -> list(emqx_types:deliver()). -callback end_trace_send(emqx_types:packet() | [emqx_types:packet()]) -> ok. -callback event(EventName :: term(), Attributes :: term()) -> ok. +-type channel_info() :: #{atom() => _}. + -export([ provider/0, register_provider/1, @@ -38,6 +40,8 @@ event/2 ]). +-export_type([channel_info/0]). + -define(PROVIDER, {?MODULE, trace_provider}). -define(with_provider(IfRegistered, IfNotRegistered), @@ -79,17 +83,17 @@ provider() -> %% trace API %%-------------------------------------------------------------------- --spec trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when +-spec trace_process_publish(Packet, ChannelInfo, fun((Packet) -> Res)) -> Res when Packet :: emqx_types:packet(), - Channel :: emqx_channel:channel(), + ChannelInfo :: channel_info(), Res :: term(). -trace_process_publish(Packet, Channel, ProcessFun) -> - ?with_provider(?FUNCTION_NAME(Packet, Channel, ProcessFun), ProcessFun(Packet, Channel)). +trace_process_publish(Packet, ChannelInfo, ProcessFun) -> + ?with_provider(?FUNCTION_NAME(Packet, ChannelInfo, ProcessFun), ProcessFun(Packet)). --spec start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> +-spec start_trace_send(list(emqx_types:deliver()), channel_info()) -> list(emqx_types:deliver()). -start_trace_send(Delivers, Channel) -> - ?with_provider(?FUNCTION_NAME(Delivers, Channel), Delivers). +start_trace_send(Delivers, ChannelInfo) -> + ?with_provider(?FUNCTION_NAME(Delivers, ChannelInfo), Delivers). -spec end_trace_send(emqx_types:packet() | [emqx_types:packet()]) -> ok. end_trace_send(Packets) -> diff --git a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl index 061c3e983..a3c73f206 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_trace.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_trace.erl @@ -94,17 +94,17 @@ stop() -> %% trace API %%-------------------------------------------------------------------- --spec trace_process_publish(Packet, Channel, fun((Packet, Channel) -> Res)) -> Res when +-spec trace_process_publish(Packet, ChannelInfo, fun((Packet) -> Res)) -> Res when Packet :: emqx_types:packet(), - Channel :: emqx_channel:channel(), + ChannelInfo :: emqx_external_trace:channel_info(), Res :: term(). -trace_process_publish(Packet, Channel, ProcessFun) -> +trace_process_publish(Packet, ChannelInfo, ProcessFun) -> case maybe_init_ctx(Packet) of false -> - ProcessFun(Packet, Channel); + ProcessFun(Packet); RootCtx -> RootCtx1 = otel_ctx:set_value(RootCtx, ?IS_ENABLED, true), - Attrs = maps:merge(packet_attributes(Packet), channel_attributes(Channel)), + Attrs = maps:merge(packet_attributes(Packet), channel_attributes(ChannelInfo)), SpanCtx = otel_tracer:start_span(RootCtx1, ?current_tracer, process_message, #{ attributes => Attrs }), @@ -113,22 +113,22 @@ trace_process_publish(Packet, Channel, ProcessFun) -> Packet1 = put_ctx_to_packet(Ctx, Packet), _ = otel_ctx:attach(Ctx), try - ProcessFun(Packet1, Channel) + ProcessFun(Packet1) after _ = ?end_span(), clear() end end. --spec start_trace_send(list(emqx_types:deliver()), emqx_channel:channel()) -> +-spec start_trace_send(list(emqx_types:deliver()), emqx_external_trace:channel_info()) -> list(emqx_types:deliver()). -start_trace_send(Delivers, Channel) -> +start_trace_send(Delivers, ChannelInfo) -> lists:map( fun({deliver, Topic, Msg} = Deliver) -> case get_ctx_from_msg(Msg) of Ctx when is_map(Ctx) -> Attrs = maps:merge( - msg_attributes(Msg), sub_channel_attributes(Channel) + msg_attributes(Msg), sub_channel_attributes(ChannelInfo) ), StartOpts = #{attributes => Attrs}, SpanCtx = otel_tracer:start_span( @@ -216,11 +216,11 @@ msg_attributes(Msg) -> packet_attributes(#mqtt_packet{variable = Packet}) -> #{'messaging.destination.name' => emqx_packet:info(topic_name, Packet)}. -channel_attributes(Channel) -> - #{'messaging.client_id' => emqx_channel:info(clientid, Channel)}. +channel_attributes(ChannelInfo) -> + #{'messaging.client_id' => maps:get(clientid, ChannelInfo, undefined)}. -sub_channel_attributes(Channel) -> - channel_attributes(Channel). +sub_channel_attributes(ChannelInfo) -> + channel_attributes(ChannelInfo). put_ctx_to_msg(OtelCtx, Msg = #message{extra = Extra}) when is_map(Extra) -> Msg#message{extra = Extra#{?EMQX_OTEL_CTX => OtelCtx}}; From 6f5228e99122b4e7c7ca48ce550e18fc099fbf20 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 5 Dec 2023 19:12:53 +0100 Subject: [PATCH 22/46] test(emqx): switch select test suites to use `emqx_cth_suite` --- apps/emqx/test/emqx_mqtt_SUITE.erl | 10 +++--- apps/emqx/test/emqx_mqtt_caps_SUITE.erl | 9 +++-- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 33 +++++-------------- apps/emqx/test/emqx_olp_SUITE.erl | 7 ++-- apps/emqx/test/emqx_os_mon_SUITE.erl | 9 +++-- 5 files changed, 24 insertions(+), 44 deletions(-) diff --git a/apps/emqx/test/emqx_mqtt_SUITE.erl b/apps/emqx/test/emqx_mqtt_SUITE.erl index f03c7af83..591a08e9a 100644 --- a/apps/emqx/test/emqx_mqtt_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -39,12 +38,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index e97684b74..8be5564b2 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -26,12 +26,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]), - ok. +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). t_check_pub(_) -> OldConf = emqx:get_config([zones], #{}), diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index a2a2e5244..ff248a16a 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -59,31 +58,17 @@ groups() -> ]. init_per_group(tcp, Config) -> - emqx_common_test_helpers:start_apps([]), - [{port, 1883}, {conn_fun, connect} | Config]; + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{port, 1883}, {conn_fun, connect}, {group_apps, Apps} | Config]; init_per_group(quic, Config) -> - UdpPort = 1884, - emqx_common_test_helpers:start_apps([]), - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), - [{port, UdpPort}, {conn_fun, quic_connect} | Config]; -init_per_group(_, Config) -> - emqx_common_test_helpers:stop_apps([]), - Config. + Apps = emqx_cth_suite:start( + [{emqx, "listeners.quic.test { enable = true, bind = 1884 }"}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{port, 1884}, {conn_fun, quic_connect}, {group_apps, Apps} | Config]. -end_per_group(quic, _Config) -> - emqx_config:put([listeners, quic], #{}), - ok; -end_per_group(_Group, _Config) -> - ok. - -init_per_suite(Config) -> - %% Start Apps - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. - -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_group(_Group, Config) -> + emqx_cth_suite:stop(?config(group_apps, Config)). init_per_testcase(TestCase, Config) -> case erlang:function_exported(?MODULE, TestCase, 2) of diff --git a/apps/emqx/test/emqx_olp_SUITE.erl b/apps/emqx/test/emqx_olp_SUITE.erl index cd8db7a8f..7389b259c 100644 --- a/apps/emqx/test/emqx_olp_SUITE.erl +++ b/apps/emqx/test/emqx_olp_SUITE.erl @@ -26,14 +26,13 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), OldSch = erlang:system_flag(schedulers_online, 1), - [{old_sch, OldSch} | Config]. + [{apps, Apps}, {old_sch, OldSch} | Config]. end_per_suite(Config) -> erlang:system_flag(schedulers_online, ?config(old_sch, Config)), - emqx_common_test_helpers:stop_apps([]). + emqx_cth_suite:stop(?config(apps, Config)). init_per_testcase(_, Config) -> emqx_common_test_helpers:boot_modules(all), diff --git a/apps/emqx/test/emqx_os_mon_SUITE.erl b/apps/emqx/test/emqx_os_mon_SUITE.erl index 1833be48e..2d7558392 100644 --- a/apps/emqx/test/emqx_os_mon_SUITE.erl +++ b/apps/emqx/test/emqx_os_mon_SUITE.erl @@ -24,12 +24,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:boot_modules(all), - emqx_common_test_helpers:start_apps([]), - Config. + Apps = emqx_cth_suite:start([emqx], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([]). +end_per_suite(Config) -> + emqx_cth_suite:stop(proplists:get_value(apps, Config)). init_per_testcase(t_cpu_check_alarm, Config) -> SysMon = emqx_config:get([sysmon, os], #{}), From 83bea2254d280ea88101d813cfd79167fd3bab11 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 5 Dec 2023 19:13:50 +0100 Subject: [PATCH 23/46] test(ocsp): switch test suite to use `emqx_cth_suite` And simplify it slightly in the process. --- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 204 +++++++----------- .../openssl_listeners.conf | 14 -- 2 files changed, 72 insertions(+), 146 deletions(-) delete mode 100644 apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index aa5d78121..fce74785c 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -44,14 +44,33 @@ groups() -> ]. init_per_suite(Config) -> - application:load(emqx), - emqx_config:save_schema_mod_and_names(emqx_schema), - emqx_common_test_helpers:boot_modules(all), Config. end_per_suite(_Config) -> ok. +init_per_group(openssl, Config) -> + DataDir = ?config(data_dir, Config), + ListenerConf = #{ + bind => <<"0.0.0.0:8883">>, + max_connections => 512000, + ssl_options => #{ + keyfile => filename(DataDir, "server.key"), + certfile => filename(DataDir, "server.pem"), + cacertfile => filename(DataDir, "ca.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + issuer_pem => filename(DataDir, "ocsp-issuer.pem"), + responder_url => <<"http://127.0.0.1:9877">> + } + } + }, + Conf = #{listeners => #{ssl => #{default => ListenerConf}}}, + Apps = emqx_cth_suite:start( + [{emqx, #{config => Conf}}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{group_apps, Apps} | Config]; init_per_group(tls12, Config) -> [{tls_vsn, "-tls1_2"} | Config]; init_per_group(tls13, Config) -> @@ -63,24 +82,14 @@ init_per_group(without_status_request, Config) -> init_per_group(_Group, Config) -> Config. +end_per_group(openssl, Config) -> + emqx_cth_suite:stop(?config(group_apps, Config)); end_per_group(_Group, _Config) -> ok. init_per_testcase(t_openssl_client, Config) -> ct:timetrap({seconds, 30}), - DataDir = ?config(data_dir, Config), - Handler = fun(_) -> ok end, {OCSPResponderPort, OCSPOSPid} = setup_openssl_ocsp(Config), - ConfFilePath = filename:join([DataDir, "openssl_listeners.conf"]), - emqx_common_test_helpers:start_apps( - [], - Handler, - #{ - extra_mustache_vars => #{test_data_dir => DataDir}, - conf_file_path => ConfFilePath - } - ), - ct:sleep(1_000), [ {ocsp_responder_port, OCSPResponderPort}, {ocsp_responder_os_pid, OCSPOSPid} @@ -107,15 +116,25 @@ init_per_testcase(TestCase, Config) when {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} end ), - emqx_mgmt_api_test_util:init_suite([emqx_conf]), + Apps = emqx_cth_suite:start( + [ + emqx_conf, + emqx, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} + ), + _ = emqx_common_test_http:create_default_app(), snabbkaffe:start_trace(), - Config; + [{tc_apps, Apps} | Config]; false -> [{skip_does_not_apply, true} | Config] end; -init_per_testcase(t_ocsp_responder_error_responses, Config) -> +init_per_testcase(TC, Config) -> ct:timetrap({seconds, 30}), TestPid = self(), + DataDir = ?config(data_dir, Config), ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), meck:expect( emqx_ocsp_cache, @@ -123,90 +142,44 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> fun(URL, _HTTPTimeout) -> ct:pal("ocsp http request ~p", [URL]), TestPid ! {http_get, URL}, - persistent_term:get({?MODULE, http_response}) + persistent_term:get( + {?MODULE, http_response}, + {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} + ) end ), - DataDir = ?config(data_dir, Config), - Type = ssl, - Name = test_ocsp, - ListenerOpts = #{ - ssl_options => - #{ - certfile => filename:join(DataDir, "server.pem"), - ocsp => #{ - enable_ocsp_stapling => true, - responder_url => <<"http://localhost:9877/">>, - issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), - refresh_http_timeout => <<"15s">>, - refresh_interval => <<"1s">> - } - } - }, - Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_utils_maps:binary_key_map(Conf), - CheckedConf = hocon_tconf:check_plain(emqx_schema, ConfBin, #{ - required => false, atom_keys => false - }), - Conf2 = emqx_utils_maps:unsafe_atom_key_map(CheckedConf), - ListenerOpts2 = emqx_utils_maps:deep_get([listeners, Type, Name], Conf2), - emqx_config:put_listener_conf(Type, Name, [], ListenerOpts2), - snabbkaffe:start_trace(), - _Heir = spawn_dummy_heir(), - {ok, CachePid} = emqx_ocsp_cache:start_link(), - [ - {cache_pid, CachePid} - | Config - ]; -init_per_testcase(_TestCase, Config) -> - ct:timetrap({seconds, 10}), - TestPid = self(), - ok = meck:new(emqx_ocsp_cache, [non_strict, passthrough, no_history, no_link]), - meck:expect( - emqx_ocsp_cache, - http_get, - fun(URL, _HTTPTimeout) -> - TestPid ! {http_get, URL}, - {ok, {{"HTTP/1.0", 200, 'OK'}, [], <<"ocsp response">>}} - end - ), - snabbkaffe:start_trace(), - _Heir = spawn_dummy_heir(), - {ok, CachePid} = emqx_ocsp_cache:start_link(), - DataDir = ?config(data_dir, Config), - Type = ssl, - Name = test_ocsp, ResponderURL = <<"http://localhost:9877/">>, - ListenerOpts = #{ - ssl_options => - #{ - certfile => filename:join(DataDir, "server.pem"), - ocsp => #{ - enable_ocsp_stapling => true, - responder_url => ResponderURL, - issuer_pem => filename:join(DataDir, "ocsp-issuer.pem"), - refresh_http_timeout => <<"15s">>, - refresh_interval => <<"1s">> - } + ListenerConf = #{ + enable => false, + bind => 0, + ssl_options => #{ + certfile => filename(DataDir, "server.pem"), + ocsp => #{ + enable_ocsp_stapling => true, + responder_url => ResponderURL, + issuer_pem => filename(DataDir, "ocsp-issuer.pem"), + refresh_http_timeout => <<"15s">>, + refresh_interval => <<"1s">> } + } }, - Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_utils_maps:binary_key_map(Conf), - CheckedConf = hocon_tconf:check_plain(emqx_schema, ConfBin, #{ - required => false, atom_keys => false - }), - Conf2 = emqx_utils_maps:unsafe_atom_key_map(CheckedConf), - ListenerOpts2 = emqx_utils_maps:deep_get([listeners, Type, Name], Conf2), - emqx_config:put_listener_conf(Type, Name, [], ListenerOpts2), + Conf = #{listeners => #{ssl => #{test_ocsp => ListenerConf}}}, + Apps = emqx_cth_suite:start( + [{emqx, #{config => Conf}}], + #{work_dir => emqx_cth_suite:work_dir(TC, Config)} + ), + snabbkaffe:start_trace(), [ - {cache_pid, CachePid}, - {responder_url, ResponderURL} + {responder_url, ResponderURL}, + {tc_apps, Apps} | Config ]. +filename(Dir, Name) -> + unicode:characters_to_binary(filename:join(Dir, Name)). + end_per_testcase(t_openssl_client, Config) -> - OCSPResponderOSPid = ?config(ocsp_responder_os_pid, Config), - catch kill_pid(OCSPResponderOSPid), - emqx_common_test_helpers:stop_apps([]), + catch kill_pid(?config(ocsp_responder_os_pid, Config)), ok; end_per_testcase(TestCase, Config) when TestCase =:= t_update_listener; @@ -217,19 +190,12 @@ end_per_testcase(TestCase, Config) when true -> ok; false -> - emqx_mgmt_api_test_util:end_suite([emqx_conf]), - meck:unload([emqx_ocsp_cache]), - ok + end_per_testcase(common, Config) end; -end_per_testcase(t_ocsp_responder_error_responses, Config) -> - CachePid = ?config(cache_pid, Config), - catch gen_server:stop(CachePid), - meck:unload([emqx_ocsp_cache]), - persistent_term:erase({?MODULE, http_response}), - ok; end_per_testcase(_TestCase, Config) -> - CachePid = ?config(cache_pid, Config), - catch gen_server:stop(CachePid), + snabbkaffe:stop(), + emqx_cth_suite:stop(?config(tc_apps, Config)), + persistent_term:erase({?MODULE, http_response}), meck:unload([emqx_ocsp_cache]), ok. @@ -237,24 +203,6 @@ end_per_testcase(_TestCase, Config) -> %% Helper functions %%-------------------------------------------------------------------- -%% The real cache makes `emqx_kernel_sup' the heir to its ETS table. -%% In some tests, we don't start the full supervision tree, so we need -%% this dummy process. -spawn_dummy_heir() -> - {_, {ok, _}} = - ?wait_async_action( - spawn_link(fun() -> - true = register(emqx_kernel_sup, self()), - ?tp(heir_name_registered, #{}), - receive - stop -> ok - end - end), - #{?snk_kind := heir_name_registered}, - 1_000 - ), - ok. - does_module_exist(Mod) -> case erlang:module_loaded(Mod) of true -> @@ -416,11 +364,6 @@ do_ensure_port_open(Port, N) when N > 0 -> do_ensure_port_open(Port, N - 1) end. -get_sni_fun(ListenerID) -> - #{opts := Opts} = emqx_listeners:find_by_id(ListenerID), - SSLOpts = proplists:get_value(ssl_options, Opts), - proplists:get_value(sni_fun, SSLOpts). - openssl_version() -> Res0 = string:trim(os:cmd("openssl version"), trailing), [_, Res] = string:split(Res0, " "), @@ -516,9 +459,7 @@ t_request_ocsp_response(_Config) -> end ). -t_request_ocsp_response_restart_cache(Config) -> - process_flag(trap_exit, true), - CachePid = ?config(cache_pid, Config), +t_request_ocsp_response_restart_cache(_Config) -> ListenerID = <<"ssl:test_ocsp">>, ?check_trace( begin @@ -526,6 +467,7 @@ t_request_ocsp_response_restart_cache(Config) -> {ok, _} = emqx_ocsp_cache:fetch_response(ListenerID), ?wait_async_action( begin + CachePid = whereis(emqx_ocsp_cache), Ref = monitor(process, CachePid), exit(CachePid, kill), receive @@ -533,9 +475,7 @@ t_request_ocsp_response_restart_cache(Config) -> ok after 1_000 -> error(cache_not_killed) - end, - {ok, _} = emqx_ocsp_cache:start_link(), - ok + end end, #{?snk_kind := ocsp_cache_init} ), diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf b/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf deleted file mode 100644 index d26e12acf..000000000 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE_data/openssl_listeners.conf +++ /dev/null @@ -1,14 +0,0 @@ -listeners.ssl.default { - bind = "0.0.0.0:8883" - max_connections = 512000 - ssl_options { - keyfile = "{{ test_data_dir }}/server.key" - certfile = "{{ test_data_dir }}/server.pem" - cacertfile = "{{ test_data_dir }}/ca.pem" - ocsp { - enable_ocsp_stapling = true - issuer_pem = "{{ test_data_dir }}/ocsp-issuer.pem" - responder_url = "http://127.0.0.1:9877" - } - } -} From e03b8fd80e6e766d24039e11a91366c2de1fb176 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 5 Dec 2023 11:13:23 -0300 Subject: [PATCH 24/46] chore(postgres): prettify logged errors Fixes https://emqx.atlassian.net/browse/EMQX-11490 --- .../test/emqx_bridge_pgsql_SUITE.erl | 6 ++- apps/emqx_postgresql/src/emqx_postgresql.erl | 45 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 58aaa7d71..e72e73e34 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -604,7 +604,11 @@ t_missing_data(Config) -> #{ result := {error, - {unrecoverable_error, {error, error, <<"23502">>, not_null_violation, _, _}}} + {unrecoverable_error, #{ + error_code := <<"23502">>, + error_codename := not_null_violation, + severity := error + }}} }, Event ), diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index ce62fa30b..660e95bd6 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -383,11 +383,12 @@ get_prepared_statement(Key, #{prepares := PrepStatements}) -> on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> try ecpool:pick_and_do(PoolName, {?MODULE, Type, [NameOrSQL, Data]}, no_handover) of - {error, Reason} = Result -> + {error, Reason} -> ?tp( pgsql_connector_query_return, #{error => Reason} ), + TranslatedError = translate_to_log_context(Reason), ?SLOG( error, maps:merge( @@ -397,7 +398,7 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> type => Type, sql => NameOrSQL }, - translate_to_log_context(Reason) + TranslatedError ) ), case Reason of @@ -406,9 +407,9 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> ecpool_empty -> {error, {recoverable_error, Reason}}; {error, error, _, undefined_table, _, _} -> - {error, {unrecoverable_error, Reason}}; + {error, {unrecoverable_error, export_error(TranslatedError)}}; _ -> - Result + {error, export_error(TranslatedError)} end; Result -> ?tp( @@ -593,15 +594,16 @@ init_prepare(State = #{}) -> {ok, PrepStatements} -> State#{prepares => PrepStatements}; Error -> + TranslatedError = translate_to_log_context(Error), ?SLOG( error, maps:merge( #{msg => <<"postgresql_init_prepare_statement_failed">>}, - translate_to_log_context(Error) + TranslatedError ) ), %% mark the prepares failed - State#{prepares => Error} + State#{prepares => {error, export_error(TranslatedError)}} end. prepare_sql(#{query_templates := Templates, pool_name := PoolName}) -> @@ -652,14 +654,15 @@ prepare_sql_to_conn(Conn, [{Key, {SQL, _RowTemplate}} | Rest], Statements) when ), ?SLOG(error, LogMsg), {error, undefined_table}; - {error, Error} = Other -> + {error, Error} -> + TranslatedError = translate_to_log_context(Error), LogMsg = maps:merge( LogMeta#{msg => "postgresql_parse_failed"}, - translate_to_log_context(Error) + TranslatedError ), ?SLOG(error, LogMsg), - Other + {error, export_error(TranslatedError)} end. to_bin(Bin) when is_binary(Bin) -> @@ -674,17 +677,21 @@ handle_result({error, {unrecoverable_error, _Error}} = Res) -> handle_result({error, disconnected}) -> {error, {recoverable_error, disconnected}}; handle_result({error, Error}) -> - {error, {unrecoverable_error, Error}}; + TranslatedError = translate_to_log_context(Error), + {error, {unrecoverable_error, export_error(TranslatedError)}}; handle_result(Res) -> Res. handle_batch_result([{ok, Count} | Rest], Acc) -> handle_batch_result(Rest, Acc + Count); handle_batch_result([{error, Error} | _Rest], _Acc) -> - {error, {unrecoverable_error, Error}}; + TranslatedError = translate_to_log_context(Error), + {error, {unrecoverable_error, export_error(TranslatedError)}}; handle_batch_result([], Acc) -> {ok, Acc}. +translate_to_log_context({error, Reason}) -> + translate_to_log_context(Reason); translate_to_log_context(#error{} = Reason) -> #error{ severity = Severity, @@ -702,3 +709,19 @@ translate_to_log_context(#error{} = Reason) -> }; translate_to_log_context(Reason) -> #{reason => Reason}. + +export_error(#{ + driver_severity := Severity, + driver_error_codename := Codename, + driver_error_code := Code +}) -> + %% Extra information has already been logged. + #{ + error_code => Code, + error_codename => Codename, + severity => Severity + }; +export_error(#{reason := Reason}) -> + Reason; +export_error(Error) -> + Error. From b015d08cbaa06dd00cb4f467ebb66c879d1d4a36 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 6 Dec 2023 08:56:20 +0800 Subject: [PATCH 25/46] fix: use sync_transation when update admin/token --- apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 14 +++++++++----- apps/emqx_dashboard/src/emqx_dashboard_token.erl | 6 +++--- apps/emqx_management/src/emqx_mgmt_auth.erl | 2 +- changes/ce/fix-12111.en.md | 1 + 4 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 changes/ce/fix-12111.en.md diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index c264a1b0f..6c2271aee 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -114,7 +114,7 @@ add_user(Username, Password, Role, Desc) when is_binary(Username), is_binary(Pas end. do_add_user(Username, Password, Role, Desc) -> - Res = mria:transaction(?DASHBOARD_SHARD, fun add_user_/4, [Username, Password, Role, Desc]), + Res = mria:sync_transaction(?DASHBOARD_SHARD, fun add_user_/4, [Username, Password, Role, Desc]), return(Res). %% 0-9 or A-Z or a-z or $_ @@ -191,7 +191,7 @@ force_add_user(Username, Password, Role, Desc) -> description = Desc }) end, - case mria:transaction(?DASHBOARD_SHARD, AddFun) of + case mria:sync_transaction(?DASHBOARD_SHARD, AddFun) of {atomic, ok} -> ok; {aborted, Reason} -> {error, Reason} end. @@ -227,7 +227,7 @@ remove_user(Username) -> _ -> mnesia:delete({?ADMIN, Username}) end end, - case return(mria:transaction(?DASHBOARD_SHARD, Trans)) of + case return(mria:sync_transaction(?DASHBOARD_SHARD, Trans)) of {ok, Result} -> _ = emqx_dashboard_token:destroy_by_username(Username), {ok, Result}; @@ -242,7 +242,11 @@ update_user(Username, Role, Desc) -> ok -> case return( - mria:transaction(?DASHBOARD_SHARD, fun update_user_/3, [Username, Role, Desc]) + mria:sync_transaction( + ?DASHBOARD_SHARD, + fun update_user_/3, + [Username, Role, Desc] + ) ) of {ok, {true, Result}} -> @@ -324,7 +328,7 @@ update_pwd(Username, Fun) -> end, mnesia:write(Fun(User)) end, - return(mria:transaction(?DASHBOARD_SHARD, Trans)). + return(mria:sync_transaction(?DASHBOARD_SHARD, Trans)). -spec lookup_user(dashboard_username()) -> [emqx_admin()]. lookup_user(Username) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 9a9875935..20041e393 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -119,7 +119,7 @@ do_sign(#?ADMIN{username = Username} = User, Password) -> {_, Token} = jose_jws:compact(Signed), Role = emqx_dashboard_admin:role(User), JWTRec = format(Token, Username, Role, ExpTime), - _ = mria:transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [JWTRec]), + _ = mria:sync_transaction(?DASHBOARD_SHARD, fun mnesia:write/1, [JWTRec]), {ok, Role, Token}. -spec do_verify(_, Token :: binary()) -> @@ -141,7 +141,7 @@ do_verify(Req, Token) -> do_destroy(Token) -> Fun = fun mnesia:delete/1, - {atomic, ok} = mria:transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]), + {atomic, ok} = mria:sync_transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]), ok. do_destroy_by_username(Username) -> @@ -266,7 +266,7 @@ check_rbac(_Req, JWT) -> save_new_jwt(OldJWT) -> #?ADMIN_JWT{exptime = _ExpTime, extra = _Extra, username = Username} = OldJWT, NewJWT = OldJWT#?ADMIN_JWT{exptime = jwt_expiration_time()}, - {atomic, Res} = mria:transaction( + {atomic, Res} = mria:sync_transaction( ?DASHBOARD_SHARD, fun mnesia:write/1, [NewJWT] diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index b75d2bf28..559344e2b 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -338,7 +338,7 @@ generate_unique_name(NamePrefix, ApiKey) -> <>. trans(Fun, Args) -> - case mria:transaction(?COMMON_SHARD, Fun, Args) of + case mria:sync_transaction(?COMMON_SHARD, Fun, Args) of {atomic, Res} -> {ok, Res}; {aborted, Error} -> {error, Error} end. diff --git a/changes/ce/fix-12111.en.md b/changes/ce/fix-12111.en.md new file mode 100644 index 000000000..b41598394 --- /dev/null +++ b/changes/ce/fix-12111.en.md @@ -0,0 +1 @@ +Fix an issue where API tokens were sometimes unavailable by using sync_transaction function to ensure all updates are consistently synchronized to the replica node. From d0732aa2a6ff4f0f1684903c54910e187e0bcbb1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 08:36:56 +0100 Subject: [PATCH 26/46] refactor: delete UDP multicast clustering strategy --- apps/emqx_conf/src/emqx_conf_cli.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 95 +------------------------ rel/i18n/emqx_conf_schema.hocon | 59 +-------------- scripts/spellcheck/dicts/emqx.txt | 1 - 4 files changed, 3 insertions(+), 154 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 7e55ada4f..b1970997f 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -468,7 +468,7 @@ fill_defaults(Conf) -> Conf1 = emqx_config:fill_defaults(Conf), filter_cluster_conf(Conf1). --define(ALL_STRATEGY, [<<"manual">>, <<"static">>, <<"dns">>, <<"etcd">>, <<"k8s">>, <<"mcast">>]). +-define(ALL_STRATEGY, [<<"manual">>, <<"static">>, <<"dns">>, <<"etcd">>, <<"k8s">>]). filter_cluster_conf(#{<<"cluster">> := #{<<"discovery_strategy">> := Strategy} = Cluster} = Conf) -> Cluster1 = maps:without(lists:delete(Strategy, ?ALL_STRATEGY), Cluster), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 0db3c4a45..8fe34d642 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -154,7 +154,7 @@ fields("cluster") -> )}, {"discovery_strategy", sc( - hoconsc:enum([manual, static, dns, etcd, k8s, mcast]), + hoconsc:enum([manual, static, dns, etcd, k8s]), #{ default => manual, desc => ?DESC(cluster_discovery_strategy), @@ -208,11 +208,6 @@ fields("cluster") -> ?R_REF(cluster_static), #{} )}, - {"mcast", - sc( - ?R_REF(cluster_mcast), - #{importance => ?IMPORTANCE_HIDDEN} - )}, {"dns", sc( ?R_REF(cluster_dns), @@ -251,81 +246,6 @@ fields(cluster_static) -> } )} ]; -fields(cluster_mcast) -> - [ - {"addr", - sc( - string(), - #{ - default => <<"239.192.0.1">>, - desc => ?DESC(cluster_mcast_addr), - 'readOnly' => true - } - )}, - {"ports", - sc( - hoconsc:array(integer()), - #{ - default => [4369, 4370], - 'readOnly' => true, - desc => ?DESC(cluster_mcast_ports) - } - )}, - {"iface", - sc( - string(), - #{ - default => <<"0.0.0.0">>, - desc => ?DESC(cluster_mcast_iface), - 'readOnly' => true - } - )}, - {"ttl", - sc( - range(0, 255), - #{ - default => 255, - desc => ?DESC(cluster_mcast_ttl), - 'readOnly' => true - } - )}, - {"loop", - sc( - boolean(), - #{ - default => true, - desc => ?DESC(cluster_mcast_loop), - 'readOnly' => true - } - )}, - {"sndbuf", - sc( - emqx_schema:bytesize(), - #{ - default => <<"16KB">>, - desc => ?DESC(cluster_mcast_sndbuf), - 'readOnly' => true - } - )}, - {"recbuf", - sc( - emqx_schema:bytesize(), - #{ - default => <<"16KB">>, - desc => ?DESC(cluster_mcast_recbuf), - 'readOnly' => true - } - )}, - {"buffer", - sc( - emqx_schema:bytesize(), - #{ - default => <<"32KB">>, - desc => ?DESC(cluster_mcast_buffer), - 'readOnly' => true - } - )} - ]; fields(cluster_dns) -> [ {"name", @@ -1100,8 +1020,6 @@ desc("cluster") -> ?DESC("desc_cluster"); desc(cluster_static) -> ?DESC("desc_cluster_static"); -desc(cluster_mcast) -> - ?DESC("desc_cluster_mcast"); desc(cluster_dns) -> ?DESC("desc_cluster_dns"); desc(cluster_etcd) -> @@ -1423,17 +1341,6 @@ map(Name, Type) -> hoconsc:map(Name, Type). cluster_options(static, Conf) -> [{seeds, conf_get("cluster.static.seeds", Conf, [])}]; -cluster_options(mcast, Conf) -> - {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), - {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), - Ports = conf_get("cluster.mcast.ports", Conf), - [ - {addr, Addr}, - {ports, Ports}, - {iface, Iface}, - {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, - {loop, conf_get("cluster.mcast.loop", Conf, true)} - ]; cluster_options(dns, Conf) -> [ {name, conf_get("cluster.dns.name", Conf)}, diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 64b96541e..32828b377 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -7,12 +7,6 @@ When drop mode is activated or deactivated, a message is printed in the logs.""" common_handler_drop_mode_qlen.label: """Queue Length before Entering Drop Mode""" -cluster_mcast_addr.desc: -"""Multicast IPv4 address.""" - -cluster_mcast_addr.label: -"""Cluster Multicast Address""" - desc_cluster_dns.desc: """Service discovery via DNS SRV records.""" @@ -33,12 +27,6 @@ Note: contents of this file are secret, so it's necessary to set permissions to rpc_keyfile.label: """RPC Keyfile""" -cluster_mcast_recbuf.desc: -"""Size of the kernel-level buffer for incoming datagrams.""" - -cluster_mcast_recbuf.label: -"""Cluster Muticast Sendbuf""" - cluster_autoheal.desc: """If true, the node will try to heal network partitions automatically.""" @@ -208,12 +196,6 @@ Otherwise, the client might opt for IPv4, even if the server is on IPv6.""" rpc_ipv6_only.label: """Use IPv6 Only""" -cluster_mcast_buffer.desc: -"""Size of the user-level buffer.""" - -cluster_mcast_buffer.label: -"""Cluster Muticast Buffer""" - rpc_authentication_timeout.desc: """Timeout for the remote node authentication.""" @@ -226,12 +208,6 @@ cluster_call_retry_interval.desc: cluster_call_retry_interval.label: """Cluster Call Retry Interval""" -cluster_mcast_sndbuf.desc: -"""Size of the kernel-level buffer for outgoing datagrams.""" - -cluster_mcast_sndbuf.label: -"""Cluster Muticast Sendbuf""" - rpc_driver.desc: """Transport protocol used for inter-broker communication""" @@ -405,12 +381,6 @@ rpc_socket_sndbuf.desc: rpc_socket_sndbuf.label: """RPC Socket Sndbuf""" -cluster_mcast_ttl.desc: -"""Time-to-live (TTL) for the outgoing UDP datagrams.""" - -cluster_mcast_ttl.label: -"""Cluster Multicast TTL""" - log_file_handler_file.desc: """Name the log file.""" @@ -435,12 +405,6 @@ desc_cluster_k8s.desc: desc_cluster_k8s.label: """Cluster Kubernetes""" -desc_cluster_mcast.desc: -"""Service discovery via UDP multicast.""" - -desc_cluster_mcast.label: -"""Cluster Multicast""" - rpc_cacertfile.desc: """Path to certification authority TLS certificate file used to validate rpc.certfile.
Note: certificates of all nodes in the cluster must be signed by the same CA.""" @@ -505,12 +469,6 @@ cluster_etcd_prefix.desc: cluster_etcd_prefix.label: """Cluster Etcd Prefix""" -cluster_mcast_iface.desc: -"""Local IP address the node discovery service needs to bind to.""" - -cluster_mcast_iface.label: -"""Cluster Multicast Interface""" - log_burst_limit_window_time.desc: """See max_count.""" @@ -583,13 +541,6 @@ desc_cluster_call.desc: desc_cluster_call.label: """Cluster Call""" -cluster_mcast_ports.desc: -"""List of UDP ports used for service discovery.
-Note: probe messages are broadcast to all the specified ports.""" - -cluster_mcast_ports.label: -"""Cluster Multicast Ports""" - log_overload_kill_mem_size.desc: """Maximum memory size that the log handler process is allowed to use.""" @@ -694,12 +645,6 @@ node_crash_dump_file.desc: node_crash_dump_file.label: """Crash Dump File""" -cluster_mcast_loop.desc: -"""If true, loop UDP datagrams back to the local socket.""" - -cluster_mcast_loop.label: -"""Cluster Multicast Loop""" - rpc_socket_keepalive_interval.desc: """The interval between keepalive messages.""" @@ -803,9 +748,7 @@ cluster_discovery_strategy.desc: - static: Configure static nodes list by setting seeds in config file.
- dns: Use DNS A record to discover peer nodes.
- etcd: Use etcd to discover peer nodes.
-- k8s: Use Kubernetes API to discover peer pods. -- mcast: Deprecated since 5.1, will be removed in 5.2. - This supports discovery via UDP multicast.""" +- k8s: Use Kubernetes API to discover peer pods.""" cluster_discovery_strategy.label: """Cluster Discovery Strategy""" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index bc05df68a..9c1799a36 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -174,7 +174,6 @@ libcoap lifecycle localhost lwm -mcast mnesia mountpoint mqueue From 1df239f2161c08f40e6f8eb395727f253d4b1fd1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 08:45:42 +0100 Subject: [PATCH 27/46] docs: Add changelog for pr 12112 (drop udp mcast clustering) --- changes/ce/breaking-12112.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/breaking-12112.en.md diff --git a/changes/ce/breaking-12112.en.md b/changes/ce/breaking-12112.en.md new file mode 100644 index 000000000..57942b29a --- /dev/null +++ b/changes/ce/breaking-12112.en.md @@ -0,0 +1 @@ +Stop supporting UDP multicast based clustering strategy. From 8965aa2a21d598aba59acb8e4ed531f7679ff8a1 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 6 Dec 2023 16:58:04 +0800 Subject: [PATCH 28/46] fix: audit log format --- apps/emqx_audit/src/emqx_audit.erl | 4 ++-- apps/emqx_conf/src/emqx_conf_schema.erl | 3 ++- apps/emqx_ctl/src/emqx_ctl.app.src | 2 +- apps/emqx_ctl/src/emqx_ctl.erl | 16 +++++++++++++++- apps/emqx_dashboard/src/emqx_dashboard_audit.erl | 10 ++++++---- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index 98f4a70e8..ffbfaebb3 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -95,7 +95,7 @@ to_audit(#{from := erlang_console, function := F, args := Args}) -> http_method = <<"">>, http_request = <<"">>, duration_ms = 0, - args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args])) + args = iolist_to_binary(io_lib:format("~p: ~ts", [F, Args])) }. log(_Level, undefined, _Handler) -> @@ -141,7 +141,7 @@ handle_continue(setup, State) -> NewState = State#{role => mria_rlog:role()}, ?AUDIT(alert, #{ cmd => emqx, - args => ["start"], + args => [<<"start">>], version => emqx_release:version(), from => cli, duration_ms => 0 diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 0db3c4a45..a0f7f5099 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1520,7 +1520,8 @@ ensure_file_handlers(Conf, _Opts) -> convert_rotation(undefined, _Opts) -> undefined; convert_rotation(#{} = Rotation, _Opts) -> maps:get(<<"count">>, Rotation, 10); -convert_rotation(Count, _Opts) when is_integer(Count) -> Count. +convert_rotation(Count, _Opts) when is_integer(Count) -> Count; +convert_rotation(Count, _Opts) -> throw({"bad_rotation", Count}). ensure_unicode_path(undefined, _) -> undefined; diff --git a/apps/emqx_ctl/src/emqx_ctl.app.src b/apps/emqx_ctl/src/emqx_ctl.app.src index 8abf67eb9..29fe06ccc 100644 --- a/apps/emqx_ctl/src/emqx_ctl.app.src +++ b/apps/emqx_ctl/src/emqx_ctl.app.src @@ -1,6 +1,6 @@ {application, emqx_ctl, [ {description, "Backend for emqx_ctl script"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_ctl_app, []}}, {applications, [ diff --git a/apps/emqx_ctl/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl index 8f6b3309b..60413fb83 100644 --- a/apps/emqx_ctl/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -339,11 +339,25 @@ audit_log(Level, From, Log) -> try apply(Mod, Fun, [Level, From, normalize_audit_log_args(Log)]) catch + _:{aborted, {no_exists, emqx_audit}} -> + case Log of + #{cmd := cluster, args := ["leave"]} -> + ok; + _ -> + ?LOG_ERROR(#{ + msg => "ctl_command_crashed", + reason => "emqx_audit table not found", + log => normalize_audit_log_args(Log), + from => From + }) + end; _:Reason:Stacktrace -> ?LOG_ERROR(#{ msg => "ctl_command_crashed", stacktrace => Stacktrace, - reason => Reason + reason => Reason, + log => normalize_audit_log_args(Log), + from => From }) end end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 4b51b2cb0..957d5d0bb 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -46,6 +46,7 @@ log_meta(Meta, Req) -> true -> undefined; false -> + Code = maps:get(code, Meta), Meta1 = #{ time => logger:timestamp(), from => from(Meta), @@ -56,8 +57,8 @@ log_meta(Meta, Req) -> %% method for http filter api. http_method => Method, http_request => http_request(Meta), - http_status_code => maps:get(code, Meta), - operation_result => operation_result(Meta), + http_status_code => Code, + operation_result => operation_result(Code, Meta), node => node() }, Meta2 = maps:without([req_start, req_end, method, headers, body, bindings, code], Meta), @@ -105,8 +106,9 @@ operation_type(Meta) -> http_request(Meta) -> maps:with([method, headers, bindings, body], Meta). -operation_result(#{failure := _}) -> failure; -operation_result(_) -> success. +operation_result(Code, _) when Code >= 300 -> failure; +operation_result(_, #{failure := _}) -> failure; +operation_result(_, _) -> success. level(get, _Code) -> debug; level(_, Code) when Code >= 200 andalso Code < 300 -> info; From b82189fb4da06b490a13a97b5e7558150ff92086 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 5 Dec 2023 16:44:33 -0300 Subject: [PATCH 29/46] test(cth_peer): use an exclusive current dir for each peer --- apps/emqx/test/emqx_cth_peer.erl | 33 ++++++++++++------- .../test/emqx_mgmt_api_plugins_SUITE.erl | 11 +++++-- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/apps/emqx/test/emqx_cth_peer.erl b/apps/emqx/test/emqx_cth_peer.erl index 8b1996cbd..9ada4c6ec 100644 --- a/apps/emqx/test/emqx_cth_peer.erl +++ b/apps/emqx/test/emqx_cth_peer.erl @@ -43,17 +43,28 @@ start_link(Name, Args, Envs, Timeout) when is_atom(Name) -> do_start(Name0, Args, Envs, Timeout, Func) when is_atom(Name0) -> {Name, Host} = parse_node_name(Name0), - {ok, Pid, Node} = peer:Func(#{ - name => Name, - host => Host, - args => Args, - env => Envs, - wait_boot => Timeout, - longnames => true, - shutdown => {halt, 1000} - }), - true = register(Node, Pid), - {ok, Node}. + %% Create exclusive current directory for the node. Some configurations, like plugin + %% installation directory, are the same for the whole cluster, and nodes on the same + %% machine will step on each other's toes... + {ok, Cwd} = file:get_cwd(), + NodeCwd = filename:join([Cwd, Name]), + ok = filelib:ensure_dir(filename:join([NodeCwd, "dummy"])), + try + file:set_cwd(NodeCwd), + {ok, Pid, Node} = peer:Func(#{ + name => Name, + host => Host, + args => Args, + env => Envs, + wait_boot => Timeout, + longnames => true, + shutdown => {halt, 1000} + }), + true = register(Node, Pid), + {ok, Node} + after + file:set_cwd(Cwd) + end. stop(Node) when is_atom(Node) -> Pid = whereis(Node), diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 4909f8ce8..96be99691 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -27,6 +27,8 @@ -define(CLUSTER_API_SERVER(PORT), ("http://127.0.0.1:" ++ (integer_to_list(PORT)))). +-import(emqx_common_test_helpers, [on_exit/1]). + all() -> emqx_common_test_helpers:all(?MODULE). @@ -66,8 +68,9 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(t_cluster_update_order, Config) -> Cluster = ?config(cluster, Config), emqx_cth_cluster:stop(Cluster), - ok; + end_per_testcase(common, Config); end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), ok. t_plugins(Config) -> @@ -136,10 +139,14 @@ t_install_plugin_matching_exisiting_name(Config) -> t_bad_plugin(Config) -> DemoShDir = proplists:get_value(demo_sh_dir, Config), PackagePathOrig = get_demo_plugin_package(DemoShDir), + BackupPath = filename:join(["/tmp", [filename:basename(PackagePathOrig), ".backup"]]), + {ok, _} = file:copy(PackagePathOrig, BackupPath), + on_exit(fun() -> {ok, _} = file:rename(BackupPath, PackagePathOrig) end), PackagePath = filename:join([ filename:dirname(PackagePathOrig), "bad_plugin-1.0.0.tar.gz" ]), + on_exit(fun() -> file:delete(PackagePath) end), ct:pal("package_location:~p orig:~p", [PackagePath, PackagePathOrig]), %% rename plugin tarball file:copy(PackagePathOrig, PackagePath), @@ -358,7 +365,7 @@ cluster(TestCase, Config) -> {Node1Name, #{role => core, apps => Node1Apps, join_to => Node1}}, {emqx_mgmt_api_plugins_SUITE2, #{role => core, apps => Node2Apps, join_to => Node1}} ], - #{work_dir => filename:join(?config(priv_dir, Config), TestCase)} + #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} ). app_specs(_Config) -> From c4956d99d1393084b563ad2b5bb2f9a3527d2d88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 6 Dec 2023 13:55:13 +0100 Subject: [PATCH 30/46] fix: bump to esockd 8.9.9 improve listener robustness --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 42f769003..34eb1a763 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, - {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}}, + {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, diff --git a/mix.exs b/mix.exs index 8e1bd2b76..75d838fa6 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,7 @@ defmodule EMQXUmbrella.MixProject do {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, - {:esockd, github: "emqx/esockd", tag: "5.9.8", override: true}, + {:esockd, github: "emqx/esockd", tag: "5.9.9", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.0", override: true}, diff --git a/rebar.config b/rebar.config index 51ec23a2e..f39a99c19 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} - , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.8"}}} + , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}} From 2757b958502ab6e1792941744ba88eb5605230d4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 6 Dec 2023 10:45:19 -0300 Subject: [PATCH 31/46] fix(actions_api): fill raw config defaults before returning Fixes https://emqx.atlassian.net/browse/EMQX-11541 --- apps/emqx_bridge/src/emqx_bridge_v2_api.erl | 17 ++++++++++++++++- .../test/emqx_bridge_v2_api_SUITE.erl | 15 ++++++++++++++- .../test/emqx_connector_api_SUITE.erl | 4 ++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl index df73d1f03..677a6de55 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2_api.erl @@ -799,11 +799,12 @@ format_resource( name := Name, status := Status, error := Error, - raw_config := RawConf, + raw_config := RawConf0, resource_data := _ResourceData }, Node ) -> + RawConf = fill_defaults(Type, RawConf0), redact( maps:merge( RawConf#{ @@ -934,6 +935,20 @@ aggregate_metrics( M17 + N17 ). +fill_defaults(Type, RawConf) -> + PackedConf = pack_bridge_conf(Type, RawConf), + FullConf = emqx_config:fill_defaults(emqx_bridge_v2_schema, PackedConf, #{}), + unpack_bridge_conf(Type, FullConf). + +pack_bridge_conf(Type, RawConf) -> + #{<<"actions">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. + +unpack_bridge_conf(Type, PackedConf) -> + TypeBin = bin(Type), + #{<<"actions">> := Bridges} = PackedConf, + #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), + RawConf. + format_bridge_status_and_error(Data) -> maps:fold(fun format_resource_data/3, #{}, maps:with([status, error], Data)). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 16bbcf7a5..0c34610ea 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -304,7 +304,7 @@ t_bridges_lifecycle(Config) -> <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"connector">> := ?CONNECTOR_NAME, - <<"kafka">> := #{}, + <<"parameters">> := #{}, <<"local_topic">> := _, <<"resource_opts">> := _ }}, @@ -1138,6 +1138,19 @@ t_cluster_later_join_metrics(Config) -> ), ok. +t_raw_config_response_defaults(Config) -> + Params = maps:remove(<<"enable">>, ?KAFKA_BRIDGE(?BRIDGE_NAME)), + ?assertMatch( + {ok, 201, #{<<"enable">> := true}}, + request_json( + post, + uri([?ROOT]), + Params, + Config + ) + ), + ok. + %%% helpers listen_on_random_port() -> SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 0b4189396..c020c2504 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -766,7 +766,7 @@ t_actions_field(Config) -> <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"connector">> := Name, - <<"kafka">> := #{}, + <<"parameters">> := #{}, <<"local_topic">> := _, <<"resource_opts">> := _ }}, @@ -821,7 +821,7 @@ t_fail_delete_with_action(Config) -> <<"status">> := <<"connected">>, <<"node_status">> := [_ | _], <<"connector">> := Name, - <<"kafka">> := #{}, + <<"parameters">> := #{}, <<"local_topic">> := _, <<"resource_opts">> := _ }}, From fdeedc360e4e69de940c6e16195ef497a18b7cec Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 6 Dec 2023 10:52:48 -0300 Subject: [PATCH 32/46] fix(actions_api): add missing fields to `POST` API schema --- .../emqx_bridge/test/emqx_bridge_v2_tests.erl | 38 +++++++++++++++++-- .../src/emqx_bridge_matrix.erl | 5 ++- .../src/emqx_bridge_pgsql.erl | 18 ++++----- .../src/emqx_bridge_timescale.erl | 5 ++- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl index d85830828..9fe2d2e76 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_tests.erl @@ -25,8 +25,8 @@ non_deprecated_fields(Fields) -> [K || {K, Schema} <- Fields, not hocon_schema:is_deprecated(Schema)]. -find_resource_opts_fields(SchemaMod, FieldName) -> - Fields = hocon_schema:fields(SchemaMod, FieldName), +find_resource_opts_fields(SchemaMod, StructName) -> + Fields = hocon_schema:fields(SchemaMod, StructName), case lists:keyfind(resource_opts, 1, Fields) of false -> undefined; @@ -35,8 +35,8 @@ find_resource_opts_fields(SchemaMod, FieldName) -> end. get_resource_opts_subfields(Sc) -> - ?R_REF(SchemaModRO, FieldNameRO) = hocon_schema:field_schema(Sc, type), - ROFields = non_deprecated_fields(hocon_schema:fields(SchemaModRO, FieldNameRO)), + ?R_REF(SchemaModRO, StructNameRO) = hocon_schema:field_schema(Sc, type), + ROFields = non_deprecated_fields(hocon_schema:fields(SchemaModRO, StructNameRO)), proplists:get_keys(ROFields). %%------------------------------------------------------------------------------ @@ -107,3 +107,33 @@ connector_resource_opts_test() -> } ), ok. + +actions_api_spec_post_fields_test() -> + ?UNION(Union) = emqx_bridge_v2_schema:post_request(), + Schemas = + lists:map( + fun(?R_REF(SchemaMod, StructName)) -> + {SchemaMod, hocon_schema:fields(SchemaMod, StructName)} + end, + hoconsc:union_members(Union) + ), + MinimalFields0 = + [ + binary_to_atom(F) + || F <- emqx_bridge_v2_schema:top_level_common_action_keys(), + F =/= <<"local_topic">> + ], + MinimalFields = [type, name | MinimalFields0], + MissingFields = + lists:filtermap( + fun({SchemaMod, FieldSchemas}) -> + Missing = MinimalFields -- proplists:get_keys(FieldSchemas), + case Missing of + [] -> false; + _ -> {true, {SchemaMod, Missing}} + end + end, + Schemas + ), + ?assertEqual(#{}, maps:from_list(MissingFields)), + ok. diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl index 4f7a1a370..205edef56 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.erl @@ -23,6 +23,7 @@ ]). -define(CONNECTOR_TYPE, matrix). +-define(ACTION_TYPE, matrix). %% ------------------------------------------------------------------------------------------------- %% api @@ -44,7 +45,7 @@ namespace() -> "bridge_matrix". roots() -> []. fields("post") -> - emqx_bridge_pgsql:fields("post", matrix); + emqx_bridge_pgsql:fields("post", ?ACTION_TYPE, "config"); fields("config_connector") -> emqx_bridge_pgsql:fields("config_connector"); fields(action) -> @@ -61,7 +62,7 @@ fields("put_bridge_v2") -> fields("get_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); fields("post_bridge_v2") -> - emqx_bridge_pgsql:fields(pgsql_action); + emqx_bridge_pgsql:fields("post", ?ACTION_TYPE, pgsql_action); fields(Field) when Field == "get_connector"; Field == "put_connector"; diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl index 4c0efe269..50c322941 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.erl @@ -13,13 +13,15 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +%% `hocon_schema' API -export([ namespace/0, roots/0, fields/1, - desc/1, - fields/2 + desc/1 ]). +%% for sharing with other actions +-export([fields/3]). %% Examples -export([ @@ -33,9 +35,7 @@ values_conn_bridge_examples/2 ]). --define(PGSQL_HOST_OPTIONS, #{ - default_port => ?PGSQL_DEFAULT_PORT -}). +-define(ACTION_TYPE, pgsql). %% Hocon Schema Definitions namespace() -> "bridge_pgsql". @@ -81,7 +81,7 @@ fields("put_bridge_v2") -> fields("get_bridge_v2") -> fields(pgsql_action); fields("post_bridge_v2") -> - fields(pgsql_action); + fields("post", pgsql, pgsql_action); fields("config") -> [ {enable, hoconsc:mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, @@ -99,14 +99,14 @@ fields("config") -> (emqx_postgresql:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> - fields("post", pgsql); + fields("post", ?ACTION_TYPE, "config"); fields("put") -> fields("config"); fields("get") -> emqx_bridge_schema:status_fields() ++ fields("post"). -fields("post", Type) -> - [type_field(Type), name_field() | fields("config")]. +fields("post", Type, StructName) -> + [type_field(Type), name_field() | fields(StructName)]. type_field(Type) -> {type, hoconsc:mk(hoconsc:enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl index 5d6c5498d..0d1d80879 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.erl @@ -23,6 +23,7 @@ ]). -define(CONNECTOR_TYPE, timescale). +-define(ACTION_TYPE, timescale). %% ------------------------------------------------------------------------------------------------- %% api @@ -44,7 +45,7 @@ namespace() -> "bridge_timescale". roots() -> []. fields("post") -> - emqx_bridge_pgsql:fields("post", timescale); + emqx_bridge_pgsql:fields("post", ?ACTION_TYPE, "config"); fields("config_connector") -> emqx_postgresql_connector_schema:fields("config_connector"); fields(action) -> @@ -61,7 +62,7 @@ fields("put_bridge_v2") -> fields("get_bridge_v2") -> emqx_bridge_pgsql:fields(pgsql_action); fields("post_bridge_v2") -> - emqx_bridge_pgsql:fields(pgsql_action); + emqx_bridge_pgsql:fields("post", ?ACTION_TYPE, pgsql_action); fields(Field) when Field == "get_connector"; Field == "put_connector"; From 5a3b8988d7eeaa29b65d08e7ca0a7dd17b3480fa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 1 Dec 2023 23:38:33 +0100 Subject: [PATCH 33/46] chore: pin ssl_verify_fun 1.1.7 see: https://github.com/deadtrickster/ssl_verify_fun.erl/pull/27/files --- rebar.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rebar.config b/rebar.config index f39a99c19..310fa79a2 100644 --- a/rebar.config +++ b/rebar.config @@ -81,6 +81,8 @@ , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} + %% to keep in sync with mix.exs + , {ssl_verify_fun, "1.1.7"} %% in conflict by erlavro and rocketmq , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}} From de61d9d609e994be7c64e851e8260d21fb522f73 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 2 Dec 2023 17:14:41 +0100 Subject: [PATCH 34/46] chore: upgrade to jiffy from 1.0.5 to 1.0.6 1.0.5 has an unexported type which causes dialyzer to fail on otp 26 --- apps/emqx_utils/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config index 4c39cfe64..7aa4d34d0 100644 --- a/apps/emqx_utils/rebar.config +++ b/apps/emqx_utils/rebar.config @@ -5,7 +5,7 @@ ]}. {deps, [ - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.6"}}} ]}. {project_plugins, [erlfmt]}. diff --git a/mix.exs b/mix.exs index 75d838fa6..9c03b943f 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,7 @@ defmodule EMQXUmbrella.MixProject do {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.11", override: true}, {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}, - {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, + {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.9", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, diff --git a/rebar.config b/rebar.config index 310fa79a2..d367b6a89 100644 --- a/rebar.config +++ b/rebar.config @@ -58,7 +58,7 @@ , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.11"}}} , {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}} - , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} + , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.6"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} From 83206daa202d7a6cb19f04dd141ecf36438ba7da Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 2 Dec 2023 19:28:40 +0100 Subject: [PATCH 35/46] refactor(emqx_cm_locker): delete dead code --- apps/emqx/src/emqx_cm_locker.erl | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl index a7dfa2761..dfe8907d5 100644 --- a/apps/emqx/src/emqx_cm_locker.erl +++ b/apps/emqx/src/emqx_cm_locker.erl @@ -23,9 +23,7 @@ -export([ trans/2, - trans/3, lock/1, - lock/2, unlock/1 ]). @@ -33,19 +31,14 @@ start_link() -> ekka_locker:start_link(?MODULE). --spec trans(emqx_types:clientid(), fun(([node()]) -> any())) -> any(). -trans(ClientId, Fun) -> - trans(ClientId, Fun, undefined). - -spec trans( maybe(emqx_types:clientid()), - fun(([node()]) -> any()), - ekka_locker:piggyback() + fun(([node()]) -> any()) ) -> any(). -trans(undefined, Fun, _Piggyback) -> +trans(undefined, Fun) -> Fun([]); -trans(ClientId, Fun, Piggyback) -> - case lock(ClientId, Piggyback) of +trans(ClientId, Fun) -> + case lock(ClientId) of {true, Nodes} -> try Fun(Nodes) @@ -56,14 +49,10 @@ trans(ClientId, Fun, Piggyback) -> {error, client_id_unavailable} end. --spec lock(emqx_types:clientid()) -> ekka_locker:lock_result(). +-spec lock(emqx_types:clientid()) -> {boolean, [node() | {node(), any()}]}. lock(ClientId) -> ekka_locker:acquire(?MODULE, ClientId, strategy()). --spec lock(emqx_types:clientid(), ekka_locker:piggyback()) -> ekka_locker:lock_result(). -lock(ClientId, Piggyback) -> - ekka_locker:acquire(?MODULE, ClientId, strategy(), Piggyback). - -spec unlock(emqx_types:clientid()) -> {boolean(), [node()]}. unlock(ClientId) -> ekka_locker:release(?MODULE, ClientId, strategy()). From 71f9838fd76977bc2d7ea38a8fcf330b4e653dc6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 4 Dec 2023 21:05:41 +0100 Subject: [PATCH 36/46] chore: upgrade to emqtt 1.10.0 --- apps/emqx/rebar.config | 2 +- apps/emqx_retainer/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 34eb1a763..27c7500d7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -45,7 +45,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.0"}}} ]}, {extra_src_dirs, [{"test", [recursive]}, {"integration_test", [recursive]}]} diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index c90ba097b..33dd50a4d 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -30,7 +30,7 @@ {profiles, [ {test, [ {deps, [ - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.6"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.0"}}} ]} ]} ]}. diff --git a/mix.exs b/mix.exs index 9c03b943f..97b2dcef0 100644 --- a/mix.exs +++ b/mix.exs @@ -64,7 +64,7 @@ defmodule EMQXUmbrella.MixProject do {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, # maybe forbid to fetch quicer {:emqtt, - github: "emqx/emqtt", tag: "1.9.7", override: true, system_env: maybe_no_quic_env()}, + github: "emqx/emqtt", tag: "1.10.0", override: true, system_env: maybe_no_quic_env()}, {:rulesql, github: "emqx/rulesql", tag: "0.1.7"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index d367b6a89..bf084bbb1 100644 --- a/rebar.config +++ b/rebar.config @@ -69,7 +69,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.9.7"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.0"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.7"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} From c44d4e7bd439d1f59db86feb80c2a077954783c9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 5 Dec 2023 08:58:27 +0100 Subject: [PATCH 37/46] chore: bump erlang-rocksdb to 1.8.0-emqx-2 erlang-rocksdb-1.8.0-emqx-2 included otp26 fixes --- mix.exs | 2 +- rebar.config | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 97b2dcef0..838e2c86b 100644 --- a/mix.exs +++ b/mix.exs @@ -54,7 +54,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.9", override: true}, - {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true}, + {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true}, {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.0", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, diff --git a/rebar.config b/rebar.config index bf084bbb1..2efcb0e3b 100644 --- a/rebar.config +++ b/rebar.config @@ -28,6 +28,15 @@ {dialyzer, [ {warnings, [unmatched_returns, error_handling]}, + {exclude_mods, [emqx_exproto_v_1_connection_unary_handler_bhvr, + emqx_exproto_v_1_connection_handler_client, + emqx_exproto_v_1_connection_handler_bhvr, + emqx_exproto_v_1_connection_adapter_client, + emqx_exproto_v_1_connection_adapter_bhvr, + emqx_exproto_v_1_connection_unary_handler_client, + emqx_exhook_v_2_hook_provider_client, + emqx_exhook_v_2_hook_provider_bhvr + ]}, {plt_location, "."}, {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, @@ -61,7 +70,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.6"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}} - , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}} + , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}} From 6affda8194201f289a2e15e5c32ead20ed8ea668 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 10:37:07 +0100 Subject: [PATCH 38/46] refactor(emqx_gateway_cm): no need to keep locker pid in state For two reasons 1. Every other places computes the process register name from gwname 2. The ekka_locker api spec only accepts atom but not pid --- apps/emqx_gateway/src/emqx_gateway_cm.erl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 2c8d708df..d2b9e531c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -96,8 +96,6 @@ -record(state, { %% Gateway Name gwname :: gateway_name(), - %% ClientId Locker for CM - locker :: pid(), %% ClientId Registry server registry :: pid(), chan_pmon :: emqx_pmon:pmon() @@ -776,7 +774,8 @@ init(Options) -> {ok, Registry} = emqx_gateway_cm_registry:start_link(GwName), %% Start locker process - {ok, Locker} = ekka_locker:start_link(lockername(GwName)), + LockerName = lockername(GwName), + {ok, _LockerPid} = ekka_locker:start_link(lockername(GwName)), %% Interval update stats %% TODO: v0.2 @@ -784,7 +783,6 @@ init(Options) -> {ok, #state{ gwname = GwName, - locker = Locker, registry = Registry, chan_pmon = emqx_pmon:new() }}. @@ -812,9 +810,9 @@ handle_info( handle_info(_Info, State) -> {noreply, State}. -terminate(_Reason, #state{registry = Registry, locker = Locker}) -> +terminate(_Reason, #state{registry = Registry, gwname = GwName}) -> _ = gen_server:stop(Registry), - _ = ekka_locker:stop(Locker), + _ = ekka_locker:stop(lockername(GwName)), ok. code_change(_OldVsn, State, _Extra) -> From 423b586c56d52716a6968c751d54852d5b71280e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 3 Dec 2023 19:17:46 +0100 Subject: [PATCH 39/46] fix(dialyzer): fix some dialyzer issues found on otp 26 --- apps/emqx/src/bhvrs/emqx_config_backup.erl | 2 +- apps/emqx/src/emqx.app.src | 1 + apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/src/emqx_cm_locker.erl | 2 +- apps/emqx/src/emqx_config_handler.erl | 9 ++++---- apps/emqx/src/emqx_crl_cache.erl | 10 +++++---- .../src/emqx_esockd_htb_limiter.erl | 4 ++-- .../src/emqx_limiter/src/emqx_htb_limiter.erl | 2 +- .../src/emqx_limiter_container.erl | 2 +- .../emqx_limiter/src/emqx_limiter_manager.erl | 2 +- .../emqx_limiter/src/emqx_limiter_server.erl | 12 +++++------ .../src/emqx_limiter_server_sup.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_session.erl | 6 +++++- apps/emqx/src/emqx_types.erl | 2 ++ apps/emqx/test/emqx_cm_locker_SUITE.erl | 2 +- .../src/emqx_auto_subscribe.app.src | 2 +- .../emqx_auto_subscribe_handler.erl | 2 +- .../src/emqx_bridge_cassandra_connector.erl | 2 +- .../src/emqx_bridge_gcp_pubsub_client.erl | 7 ++++--- ...emqx_bridge_gcp_pubsub_consumer_worker.erl | 5 +++-- .../src/emqx_bridge_syskeeper_client.erl | 4 +++- .../emqx_connector/src/emqx_connector_jwt.erl | 3 ++- .../src/emqx_connector_jwt_worker.erl | 10 +++++---- .../proto/emqx_eviction_agent_proto_v2.erl | 2 +- .../src/bhvrs/emqx_gateway_channel.erl | 2 ++ .../src/bhvrs/emqx_gateway_impl.erl | 2 ++ apps/emqx_gateway/src/emqx_gateway_cm.erl | 1 - .../src/emqx_gateway_cm_registry.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_ctx.erl | 2 ++ apps/emqx_gateway/src/emqx_gateway_utils.erl | 8 +++---- .../src/proto/emqx_gateway_cm_proto_v1.erl | 2 +- apps/emqx_gateway_coap/src/emqx_coap_tm.erl | 2 +- .../src/emqx_exproto_channel.erl | 2 +- .../src/emqx_exproto_gcli.erl | 2 ++ .../src/emqx_exproto_gsvr.erl | 21 ++++++++++++------- .../src/emqx_gateway_exproto.app.src | 2 +- .../src/emqx_gateway_lwm2m.app.src | 2 +- .../src/emqx_lwm2m_xml_object_db.erl | 6 ++++-- .../src/emqx_mqttsn_registry.erl | 2 ++ .../src/emqx_ocpp_connection.erl | 6 +++--- .../emqx_gateway_ocpp/src/emqx_ocpp_frame.erl | 7 +++++-- .../src/emqx_stomp_channel.erl | 2 +- .../src/emqx_stomp_heartbeat.erl | 2 ++ apps/emqx_ldap/src/emqx_ldap.erl | 2 +- apps/emqx_ldap/src/emqx_ldap_bind_worker.erl | 2 +- apps/emqx_machine/src/emqx_global_gc.erl | 3 ++- apps/emqx_machine/src/emqx_machine.app.src | 2 +- .../src/emqx_management.app.src | 4 +++- .../src/emqx_mgmt_data_backup.erl | 2 +- apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 2 +- .../src/emqx_node_rebalance_evacuation.erl | 3 ++- apps/emqx_oracle/src/emqx_oracle.erl | 2 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 2 +- .../src/emqx_prometheus_mria.erl | 2 +- .../src/emqx_resource_buffer_worker.erl | 17 ++++++++------- .../src/schema/emqx_resource_schema.erl | 3 ++- apps/emqx_retainer/src/emqx_retainer.erl | 4 ++-- .../emqx_retainer/src/emqx_retainer_index.erl | 2 +- apps/emqx_s3/src/emqx_s3.app.src | 2 +- apps/emqx_s3/src/emqx_s3.erl | 2 +- apps/emqx_s3/src/emqx_s3_client.erl | 7 ++++++- apps/emqx_s3/src/emqx_s3_profile_sup.erl | 2 +- .../src/emqx_s3_profile_uploader_sup.erl | 4 ++-- apps/emqx_s3/src/emqx_s3_sup.erl | 2 +- apps/emqx_s3/src/emqx_s3_uploader.erl | 7 ++++++- .../emqx_utils/src/bpapi/emqx_bpapi_trans.erl | 2 +- apps/emqx_utils/src/emqx_utils.erl | 8 ++++++- 70 files changed, 163 insertions(+), 103 deletions(-) diff --git a/apps/emqx/src/bhvrs/emqx_config_backup.erl b/apps/emqx/src/bhvrs/emqx_config_backup.erl index 604fef106..69edd64c3 100644 --- a/apps/emqx/src/bhvrs/emqx_config_backup.erl +++ b/apps/emqx/src/bhvrs/emqx_config_backup.erl @@ -19,6 +19,6 @@ -callback import_config(RawConf :: map()) -> {ok, #{ root_key => emqx_utils_maps:config_key(), - changed => [emqx_utils_maps:config_path()] + changed => [emqx_utils_maps:config_key_path()] }} | {error, #{root_key => emqx_utils_maps:config_key(), reason => term()}}. diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 915a66f17..c7c5aaef2 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -11,6 +11,7 @@ gproc, gen_rpc, mria, + ekka, esockd, cowboy, sasl, diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 5069076e5..4662eaee5 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -96,7 +96,7 @@ %% Authentication Data Cache auth_cache :: maybe(map()), %% Quota checkers - quota :: emqx_limiter_container:limiter(), + quota :: emqx_limiter_container:container(), %% Timers timers :: #{atom() => disabled | maybe(reference())}, %% Conn State diff --git a/apps/emqx/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl index dfe8907d5..f56f9239a 100644 --- a/apps/emqx/src/emqx_cm_locker.erl +++ b/apps/emqx/src/emqx_cm_locker.erl @@ -49,7 +49,7 @@ trans(ClientId, Fun) -> {error, client_id_unavailable} end. --spec lock(emqx_types:clientid()) -> {boolean, [node() | {node(), any()}]}. +-spec lock(emqx_types:clientid()) -> {boolean(), [node() | {node(), any()}]}. lock(ClientId) -> ekka_locker:acquire(?MODULE, ClientId, strategy()). diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 05784feb7..4cc5b2908 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -84,6 +84,7 @@ ok | {ok, Result :: any()} | {error, Reason :: term()}. -type state() :: #{handlers := any()}. +-type config_key_path() :: emqx_utils_maps:config_key_path(). start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []). @@ -91,21 +92,21 @@ start_link() -> stop() -> gen_server:stop(?MODULE). --spec update_config(module(), emqx_config:config_key_path(), emqx_config:update_args()) -> +-spec update_config(module(), config_key_path(), emqx_config:update_args()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> %% force convert the path to a list of atoms, as there maybe some wildcard names/ids in the path AtomKeyPath = [atom(Key) || Key <- ConfKeyPath], gen_server:call(?MODULE, {change_config, SchemaModule, AtomKeyPath, UpdateArgs}, infinity). --spec add_handler(emqx_config:config_key_path(), handler_name()) -> +-spec add_handler(config_key_path(), handler_name()) -> ok | {error, {conflict, list()}}. add_handler(ConfKeyPath, HandlerName) -> assert_callback_function(HandlerName), gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}). %% @doc Remove handler asynchronously --spec remove_handler(emqx_config:config_key_path()) -> ok. +-spec remove_handler(config_key_path()) -> ok. remove_handler(ConfKeyPath) -> gen_server:cast(?MODULE, {remove_handler, ConfKeyPath}). @@ -764,7 +765,7 @@ assert_callback_function(Mod) -> end, ok. --spec schema(module(), emqx_utils_maps:config_key_path()) -> hocon_schema:schema(). +-spec schema(module(), config_key_path()) -> hocon_schema:schema(). schema(SchemaModule, [RootKey | _]) -> Roots = hocon_schema:roots(SchemaModule), {Field, Translations} = diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 0ca779181..45c76c7f4 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -58,12 +58,14 @@ -define(DEFAULT_CACHE_CAPACITY, 100). -define(CONF_KEY_PATH, [crl_cache]). +-type duration() :: non_neg_integer(). + -record(state, { - refresh_timers = #{} :: #{binary() => timer:tref()}, - refresh_interval = timer:minutes(15) :: timer:time(), - http_timeout = ?HTTP_TIMEOUT :: timer:time(), + refresh_timers = #{} :: #{binary() => reference()}, + refresh_interval = timer:minutes(15) :: duration(), + http_timeout = ?HTTP_TIMEOUT :: duration(), %% keeps track of URLs by insertion time - insertion_times = gb_trees:empty() :: gb_trees:tree(timer:time(), url()), + insertion_times = gb_trees:empty() :: gb_trees:tree(duration(), url()), %% the set of cached URLs, for testing if an URL is already %% registered. cached_urls = sets:new([{version, 2}]) :: sets:set(url()), diff --git a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl index 8b2831766..77873b6f2 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_esockd_htb_limiter.erl @@ -25,7 +25,7 @@ module := ?MODULE, id := emqx_limiter_schema:limiter_id(), type := emqx_limiter_schema:limiter_type(), - bucket := hocons:config() + bucket := hocon:config() }. %%-------------------------------------------------------------------- @@ -35,7 +35,7 @@ -spec new_create_options( emqx_limiter_schema:limiter_id(), emqx_limiter_schema:limiter_type(), - hocons:config() + hocon:config() ) -> create_options(). new_create_options(Id, Type, BucketCfg) -> #{module => ?MODULE, id => Id, type => Type, bucket => BucketCfg}. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 7f50161a8..15e838b6e 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -32,7 +32,7 @@ make_future/1, available/1 ]). --export_type([local_limiter/0]). +-export_type([local_limiter/0, limiter/0]). %% a token bucket limiter which may or not contains a reference to another limiter, %% and can be used in a client alone diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl index 6a9101a0f..fb97a9b67 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_container.erl @@ -70,7 +70,7 @@ -spec get_limiter_by_types( limiter_id() | {atom(), atom()}, list(limiter_type()), - #{limiter_type() => hocons:config()} + #{limiter_type() => hocon:config()} ) -> container(). get_limiter_by_types({Type, Listener}, Types, BucketCfgs) -> Id = emqx_listeners:listener_id(Type, Listener), diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl index 91d59b3be..5a7c3d828 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl @@ -77,7 +77,7 @@ start_server(Type) -> emqx_limiter_server_sup:start(Type). --spec start_server(limiter_type(), hocons:config()) -> _. +-spec start_server(limiter_type(), hocon:config()) -> _. start_server(Type, Cfg) -> emqx_limiter_server_sup:start(Type, Cfg). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 94bded7a9..465b3af09 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -112,7 +112,7 @@ -spec connect( limiter_id(), limiter_type(), - hocons:config() | undefined + hocon:config() | undefined ) -> {ok, emqx_htb_limiter:limiter()} | {error, _}. %% undefined is the default situation, no limiter setting by default @@ -128,7 +128,7 @@ connect(Id, Type, Cfg) -> maps:get(Type, Cfg, undefined) ). --spec add_bucket(limiter_id(), limiter_type(), hocons:config() | undefined) -> ok. +-spec add_bucket(limiter_id(), limiter_type(), hocon:config() | undefined) -> ok. add_bucket(_Id, _Type, undefined) -> ok; %% a bucket with an infinity rate shouldn't be added to this server, because it is always full @@ -153,7 +153,7 @@ name(Type) -> restart(Type) -> ?CALL(Type). --spec update_config(limiter_type(), hocons:config()) -> ok | {error, _}. +-spec update_config(limiter_type(), hocon:config()) -> ok | {error, _}. update_config(Type, Config) -> ?CALL(Type, {update_config, Type, Config}). @@ -166,7 +166,7 @@ whereis(Type) -> %% Starts the server %% @end %%-------------------------------------------------------------------- --spec start_link(limiter_type(), hocons:config()) -> _. +-spec start_link(limiter_type(), hocon:config()) -> _. start_link(Type, Cfg) -> gen_server:start_link({local, name(Type)}, ?MODULE, [Type, Cfg], []). @@ -500,7 +500,7 @@ init_tree(Type, #{rate := Rate} = Cfg) -> buckets => #{} }. --spec make_root(hocons:confg()) -> root(). +-spec make_root(hocon:config()) -> root(). make_root(#{rate := Rate, burst := Burst}) -> #{ rate => Rate, @@ -554,7 +554,7 @@ do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> State#{buckets := maps:remove(Id, Buckets)} end. --spec get_initial_val(hocons:config()) -> decimal(). +-spec get_initial_val(hocon:config()) -> decimal(). get_initial_val( #{ initial := Initial, diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl index 8f45da561..428f75c22 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl @@ -47,7 +47,7 @@ start(Type) -> Spec = make_child(Type), supervisor:start_child(?MODULE, Spec). --spec start(emqx_limiter_schema:limiter_type(), hocons:config()) -> _. +-spec start(emqx_limiter_schema:limiter_type(), hocon:config()) -> _. start(Type, Cfg) -> Spec = make_child(Type, Cfg), supervisor:start_child(?MODULE, Spec). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index cdb1035df..c6efe6e46 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3373,7 +3373,7 @@ naive_env_interpolation("$" ++ Maybe = Original) -> filename:join([Path, Tail]); error -> ?SLOG(warning, #{ - msg => "failed_to_resolve_env_variable", + msg => "cannot_resolve_env_variable", env => Env, original => Original }), diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index bcf711a76..fa9469a61 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -114,7 +114,9 @@ reply/0, replies/0, common_timer_name/0, - custom_timer_name/0 + custom_timer_name/0, + session_id/0, + session/0 ]). -type session_id() :: _TODO. @@ -150,6 +152,8 @@ await_rel_timeout := timeout() }. +-type session() :: t(). + -type t() :: emqx_session_mem:session() | emqx_persistent_session_ds:session(). diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 1a4825736..436fffe4e 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -114,6 +114,8 @@ -export_type([takeover_data/0]). +-export_type([startlink_ret/0]). + -type proto_ver() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 diff --git a/apps/emqx/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl index 3dfb6e5ad..08192d98c 100644 --- a/apps/emqx/test/emqx_cm_locker_SUITE.erl +++ b/apps/emqx/test/emqx_cm_locker_SUITE.erl @@ -35,7 +35,7 @@ t_start_link(_) -> emqx_cm_locker:start_link(). t_trans(_) -> - ok = emqx_cm_locker:trans(undefined, fun(_) -> ok end, []), + ok = emqx_cm_locker:trans(undefined, fun(_) -> ok end), ok = emqx_cm_locker:trans(<<"clientid">>, fun(_) -> ok end). t_lock_unlock(_) -> diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index 1296996b6..a6fd53ee7 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ {description, "Auto subscribe Application"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl index 5b24f106e..6b39d3b56 100644 --- a/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl +++ b/apps/emqx_auto_subscribe/src/topics_handler/emqx_auto_subscribe_handler.erl @@ -17,7 +17,7 @@ -export([init/1]). --spec init(hocons:config()) -> {Module :: atom(), Config :: term()}. +-spec init(hocon:config()) -> {Module :: atom(), Config :: term()}. init(Config) -> do_init(Config). diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index e29dc7931..c6bc7098c 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -88,7 +88,7 @@ keyspace(_) -> undefined. callback_mode() -> async_if_possible. --spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +-spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl index 454c0d7ea..5975ba89b 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl @@ -27,6 +27,7 @@ -type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json(). -type project_id() :: binary(). +-type duration() :: non_neg_integer(). -type config() :: #{ connect_timeout := emqx_schema:duration_ms(), max_retries := non_neg_integer(), @@ -35,12 +36,12 @@ any() => term() }. -opaque state() :: #{ - connect_timeout := timer:time(), + connect_timeout := duration(), jwt_config := emqx_connector_jwt:jwt_config(), max_retries := non_neg_integer(), pool_name := binary(), project_id := project_id(), - request_ttl := infinity | timer:time() + request_ttl := erlang:timeout() }. -type headers() :: [{binary(), iodata()}]. -type body() :: iodata(). @@ -414,7 +415,7 @@ reply_delegator(ResourceId, ReplyFunAndArgs, Response) -> Result = handle_response(Response, ResourceId, _QueryMode = async), emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result). --spec do_get_status(resource_id(), timer:time()) -> boolean(). +-spec do_get_status(resource_id(), duration()) -> boolean(). do_get_status(ResourceId, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ehttpc:workers(ResourceId)], DoPerWorker = diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl index 44b2d022a..93f8fd8c3 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl @@ -30,12 +30,13 @@ -type bridge_name() :: atom() | binary(). -type ack_id() :: binary(). -type message_id() :: binary(). +-type duration() :: non_neg_integer(). -type config() :: #{ ack_deadline := emqx_schema:timeout_duration_s(), ack_retry_interval := emqx_schema:timeout_duration_ms(), client := emqx_bridge_gcp_pubsub_client:state(), ecpool_worker_id => non_neg_integer(), - forget_interval := timer:time(), + forget_interval := duration(), hookpoint := binary(), instance_id := binary(), mqtt_config => emqx_bridge_gcp_pubsub_impl_consumer:mqtt_config(), @@ -52,7 +53,7 @@ async_workers := #{pid() => reference()}, client := emqx_bridge_gcp_pubsub_client:state(), ecpool_worker_id := non_neg_integer(), - forget_interval := timer:time(), + forget_interval := duration(), hookpoint := binary(), instance_id := binary(), mqtt_config := emqx_bridge_gcp_pubsub_impl_consumer:mqtt_config(), diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl index 18822886f..cfb8dddfe 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl @@ -25,9 +25,11 @@ -include("emqx_bridge_syskeeper.hrl"). +-type duration() :: non_neg_integer(). + -type state() :: #{ ack_mode := need_ack | no_ack, - ack_timeout := timer:time(), + ack_timeout := duration(), socket := undefined | inet:socket(), frame_state := emqx_bridge_syskeeper_frame:state(), last_error := undefined | tuple() diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector/src/emqx_connector_jwt.erl index 49b7b3a0e..9945aa148 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt.erl @@ -33,8 +33,9 @@ -type jwt() :: binary(). -type wrapped_jwk() :: fun(() -> jose_jwk:key()). -type jwk() :: jose_jwk:key(). +-type duration() :: non_neg_integer(). -type jwt_config() :: #{ - expiration := timer:time(), + expiration := duration(), resource_id := resource_id(), table := ets:table(), jwk := wrapped_jwk() | jwk(), diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index c814b32e1..316d259d3 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -41,10 +41,12 @@ -include_lib("jose/include/jose_jwk.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-type duration() :: non_neg_integer(). + -type config() :: #{ private_key := binary(), resource_id := resource_id(), - expiration := timer:time(), + expiration := duration(), table := ets:table(), iss := binary(), sub := binary(), @@ -54,9 +56,9 @@ }. -type jwt() :: binary(). -type state() :: #{ - refresh_timer := undefined | timer:tref() | reference(), + refresh_timer := undefined | reference(), resource_id := resource_id(), - expiration := timer:time(), + expiration := duration(), table := ets:table(), jwt := undefined | jwt(), %% only undefined during startup @@ -221,7 +223,7 @@ censor_secret(undefined) -> censor_secret(_Secret) -> "******". --spec cancel_timer(undefined | timer:tref() | reference()) -> ok. +-spec cancel_timer(undefined | reference() | reference()) -> ok. cancel_timer(undefined) -> ok; cancel_timer(TRef) -> diff --git a/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl b/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl index 2d204079c..128ad5cf2 100644 --- a/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl +++ b/apps/emqx_eviction_agent/src/proto/emqx_eviction_agent_proto_v2.erl @@ -30,6 +30,6 @@ evict_session_channel(Node, ClientId, ConnInfo, ClientInfo) -> rpc:call(Node, emqx_eviction_agent, evict_session_channel, [ClientId, ConnInfo, ClientInfo]). %% Introduced in v2: --spec all_channels_count([node()], time:time()) -> emqx_rpc:erpc_multicall(non_neg_integer()). +-spec all_channels_count([node()], timeout()) -> emqx_rpc:erpc_multicall(non_neg_integer()). all_channels_count(Nodes, Timeout) -> erpc:multicall(Nodes, emqx_eviction_agent, all_local_channels_count, [], Timeout). diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl index c1a5ea2e8..4d77138cf 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -20,6 +20,8 @@ %% module if it integrated with emqx_gateway_conn module -module(emqx_gateway_channel). +-export_type([gen_server_from/0]). + -type channel() :: any(). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl index d8cb871ef..b3017ec99 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl @@ -16,6 +16,8 @@ -module(emqx_gateway_impl). +-export_type([state/0]). + -include("emqx_gateway.hrl"). -type state() :: map(). diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index d2b9e531c..d5968439b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -774,7 +774,6 @@ init(Options) -> {ok, Registry} = emqx_gateway_cm_registry:start_link(GwName), %% Start locker process - LockerName = lockername(GwName), {ok, _LockerPid} = ekka_locker:start_link(lockername(GwName)), %% Interval update stats diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl index f7a72af5f..5151a8706 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl @@ -53,7 +53,7 @@ -record(channel, {chid, pid}). %% @doc Start the global channel registry for the given gateway name. --spec start_link(gateway_name()) -> gen_server:startlink_ret(). +-spec start_link(gateway_name()) -> emqx_types:startlink_ret(). start_link(Name) -> gen_server:start_link(?MODULE, [Name], []). diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 6df1a8aae..4c630c39d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -17,6 +17,8 @@ %% @doc The gateway instance context -module(emqx_gateway_ctx). +-export_type([context/0]). + -include("emqx_gateway.hrl"). %% @doc The running context for a Connection/Channel process. diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index ed3f10594..eef1a42fb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -89,17 +89,17 @@ -elvis([{elvis_style, god_modules, disable}]). --spec childspec(supervisor:worker(), Mod :: atom()) -> +-spec childspec(worker | supervisor, Mod :: atom()) -> supervisor:child_spec(). childspec(Type, Mod) -> childspec(Mod, Type, Mod, []). --spec childspec(supervisor:worker(), Mod :: atom(), Args :: list()) -> +-spec childspec(worker | supervisor, Mod :: atom(), Args :: list()) -> supervisor:child_spec(). childspec(Type, Mod, Args) -> childspec(Mod, Type, Mod, Args). --spec childspec(atom(), supervisor:worker(), Mod :: atom(), Args :: list()) -> +-spec childspec(atom(), worker | supervisor, Mod :: atom(), Args :: list()) -> supervisor:child_spec(). childspec(Id, Type, Mod, Args) -> #{ @@ -121,7 +121,7 @@ supervisor_ret({error, {Reason, Child}}) -> supervisor_ret(Ret) -> Ret. --spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id()) -> +-spec find_sup_child(Sup :: pid() | atom(), ChildId :: term()) -> false | {ok, pid()}. find_sup_child(Sup, ChildId) -> diff --git a/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl b/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl index 29b6c7486..74e973d42 100644 --- a/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl +++ b/apps/emqx_gateway/src/proto/emqx_gateway_cm_proto_v1.erl @@ -40,7 +40,7 @@ introduced_in() -> "5.0.0". -spec lookup_by_clientid([node()], emqx_gateway_cm:gateway_name(), emqx_types:clientid()) -> - emqx_rpc:multicall_return([pid()]). + emqx_rpc:multicall_result([pid()]). lookup_by_clientid(Nodes, GwName, ClientId) -> rpc:multicall(Nodes, emqx_gateway_cm, do_lookup_by_clientid, [GwName, ClientId]). diff --git a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl index 68a7ae237..01a7684e9 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl @@ -40,7 +40,7 @@ token :: token() | undefined, observe :: 0 | 1 | undefined | observed, state :: atom(), - timers :: maps:map(), + timers :: map(), transport :: emqx_coap_transport:transport() }). -type state_machine() :: #state_machine{}. diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index a1d598923..1b3d057a8 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -131,7 +131,7 @@ stats(#channel{subscriptions = Subs}) -> %% Init the channel %%-------------------------------------------------------------------- --spec init(emqx_exproto_types:conninfo(), map()) -> channel(). +-spec init(emqx_types:conninfo(), map()) -> channel(). init( ConnInfo = #{ socktype := Socktype, diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl index 639e3bfe9..f8d049cd1 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl @@ -29,6 +29,8 @@ is_empty/1 ]). +-export_type([grpc_client_state/0]). + -define(CONN_HANDLER_MOD, emqx_exproto_v_1_connection_handler_client). -define(CONN_UNARY_HANDLER_MOD, emqx_exproto_v_1_connection_unary_handler_client). diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl index 043c910da..6a8ed0d53 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl @@ -38,13 +38,18 @@ unsubscribe/2 ]). +%% TODO: +%% The spec should be :: grpc_cowboy_h:error_response() +%% But there is no such module as grpc_cowboy_h +-type error_response() :: term(). + %%-------------------------------------------------------------------- %% gRPC ConnectionAdapter service %%-------------------------------------------------------------------- -spec send(emqx_exproto_pb:send_bytes_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. send(Req = #{conn := Conn, bytes := Bytes}, Md) -> ?SLOG(debug, #{ msg => "recv_grpc_function_call", @@ -55,7 +60,7 @@ send(Req = #{conn := Conn, bytes := Bytes}, Md) -> -spec close(emqx_exproto_pb:close_socket_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. close(Req = #{conn := Conn}, Md) -> ?SLOG(debug, #{ msg => "recv_grpc_function_call", @@ -66,7 +71,7 @@ close(Req = #{conn := Conn}, Md) -> -spec authenticate(emqx_exproto_pb:authenticate_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. authenticate( Req = #{ conn := Conn, @@ -89,7 +94,7 @@ authenticate( -spec start_timer(emqx_exproto_pb:timer_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. start_timer(Req = #{conn := Conn, type := Type, interval := Interval}, Md) when Type =:= 'KEEPALIVE' andalso Interval > 0 -> @@ -111,7 +116,7 @@ start_timer(Req, Md) -> -spec publish(emqx_exproto_pb:publish_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. publish(Req = #{conn := Conn, topic := Topic, qos := Qos, payload := Payload}, Md) when ?IS_QOS(Qos) -> @@ -132,7 +137,7 @@ publish(Req, Md) -> -spec raw_publish(emqx_exproto_pb:raw_publish_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_stream:error_response()}. + | {error, error_response()}. raw_publish(Req = #{topic := Topic, qos := Qos, payload := Payload}, Md) -> ?SLOG(debug, #{ msg => "recv_grpc_function_call", @@ -145,7 +150,7 @@ raw_publish(Req = #{topic := Topic, qos := Qos, payload := Payload}, Md) -> -spec subscribe(emqx_exproto_pb:subscribe_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. subscribe(Req = #{conn := Conn, topic := Topic, qos := Qos}, Md) when ?IS_QOS(Qos) -> @@ -165,7 +170,7 @@ subscribe(Req, Md) -> -spec unsubscribe(emqx_exproto_pb:unsubscribe_request(), grpc:metadata()) -> {ok, emqx_exproto_pb:code_response(), grpc:metadata()} - | {error, grpc_cowboy_h:error_response()}. + | {error, error_response()}. unsubscribe(Req = #{conn := Conn, topic := Topic}, Md) -> ?SLOG(debug, #{ msg => "recv_grpc_function_call", diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 2bdd0956e..3ce320d2d 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index f4ab5bd24..36b6bcf4f 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -3,7 +3,7 @@ {description, "LwM2M Gateway"}, {vsn, "0.1.5"}, {registered, []}, - {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, + {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap, xmerl]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 2908a65e0..d7dd0e655 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -43,6 +43,8 @@ -define(LWM2M_OBJECT_DEF_TAB, lwm2m_object_def_tab). -define(LWM2M_OBJECT_NAME_TO_ID_TAB, lwm2m_object_name_to_id_tab). +-type xmlElement() :: tuple(). + -record(state, {}). -elvis([{elvis_style, atom_naming_convention, disable}]). @@ -59,7 +61,7 @@ start_link(XmlDir) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []). --spec find_objectid(integer()) -> {error, no_xml_definition} | xmerl:xmlElement(). +-spec find_objectid(integer()) -> {error, no_xml_definition} | xmlElement(). find_objectid(ObjectId) -> ObjectIdInt = case is_list(ObjectId) of @@ -71,7 +73,7 @@ find_objectid(ObjectId) -> [{_ObjectId, Xml}] -> Xml end. --spec find_name(string()) -> {error, no_xml_definition} | xmerl:xmlElement(). +-spec find_name(string()) -> {error, no_xml_definition} | xmlElement(). find_name(Name) -> NameBinary = case is_list(Name) of diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl index 3113fc43d..6c397d6ac 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl @@ -32,6 +32,8 @@ lookup_topic_id/2 ]). +-export_type([registry/0]). + -define(PKEY(Id), {mqttsn, predef_topics, Id}). -type registry() :: #{ diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl index 51389f6e4..6eadb169c 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_connection.erl @@ -72,13 +72,13 @@ %% Piggyback piggyback :: single | multiple, %% Limiter - limiter :: maybe(emqx_limiter:limiter()), + limiter :: maybe(emqx_htb_limiter:limiter()), %% Limit Timer limit_timer :: maybe(reference()), %% Parse State parse_state :: emqx_ocpp_frame:parse_state(), %% Serialize options - serialize :: emqx_ocpp_frame:serialize_opts(), + serialize :: emqx_ocpp_frame:serialize_options(), %% Channel channel :: emqx_ocpp_channel:channel(), %% GC State @@ -268,7 +268,7 @@ init_state_and_channel([Req, Opts, _WsOpts], _State = undefined) -> ws_cookie => WsCookie, conn_mod => ?MODULE }, - Limiter = undeined, + Limiter = undefined, ActiveN = emqx_gateway_utils:active_n(Opts), Piggyback = emqx_utils_maps:deep_get([websocket, piggyback], Opts, multiple), ParseState = emqx_ocpp_frame:initial_parse_state(#{}), diff --git a/apps/emqx_gateway_ocpp/src/emqx_ocpp_frame.erl b/apps/emqx_gateway_ocpp/src/emqx_ocpp_frame.erl index d404067e1..a3f6303c3 100644 --- a/apps/emqx_gateway_ocpp/src/emqx_ocpp_frame.erl +++ b/apps/emqx_gateway_ocpp/src/emqx_ocpp_frame.erl @@ -39,9 +39,12 @@ -export_type([ parse_state/0, parse_result/0, - frame/0 + frame/0, + serialize_options/0 ]). +-type serialize_options() :: emqx_gateway_frame:serialize_options(). + -dialyzer({nowarn_function, [format/1]}). -spec initial_parse_state(map()) -> parse_state(). @@ -114,7 +117,7 @@ parse( }, <<>>, Parser}. --spec serialize_opts() -> emqx_gateway_frame:serialize_options(). +-spec serialize_opts() -> serialize_options(). serialize_opts() -> #{}. diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 10d081e57..472a68200 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -69,7 +69,7 @@ %% Channel State conn_state :: conn_state(), %% Heartbeat - heartbeat :: emqx_stomp_heartbeat:heartbeat(), + heartbeat :: undefined | emqx_stomp_heartbeat:heartbeat(), %% Subscriptions subscriptions = [], %% Timer diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl index 2e4239bdc..98ea481d9 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl @@ -27,6 +27,8 @@ interval/2 ]). +-export_type([heartbeat/0]). + -record(heartbeater, {interval, statval, repeat}). -type name() :: incoming | outgoing. diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 1ff6861ed..04b61918a 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -130,7 +130,7 @@ ensure_username(Field) -> %% =================================================================== callback_mode() -> always_sync. --spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +-spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ diff --git a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl index 834cbac5a..e19818893 100644 --- a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl +++ b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl @@ -33,7 +33,7 @@ -define(POOL_NAME_SUFFIX, "bind_worker"). %% =================================================================== --spec on_start(binary(), hoconsc:config(), proplists:proplist(), map()) -> +-spec on_start(binary(), hocon:config(), proplists:proplist(), map()) -> {ok, binary(), map()} | {error, _}. on_start(InstId, #{method := #{bind_password := _}} = Config, Options, State) -> PoolName = pool_name(InstId), diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index 121855e68..4f7f9e086 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -43,7 +43,8 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec run() -> {ok, timer:time()}. +%% @doc Run global garbage collection and return the time (in milliseconds) spent. +-spec run() -> {ok, non_neg_integer()}. run() -> gen_server:call(?MODULE, run, infinity). -spec stop() -> ok. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 6d7012313..904d5fe77 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -6,7 +6,7 @@ {vsn, "0.2.17"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib, emqx_ctl]}, + {applications, [kernel, stdlib, emqx_ctl, system_monitor, covertool]}, {mod, {emqx_machine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index f9deaf819..0aa522d23 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -5,7 +5,9 @@ {vsn, "5.0.34"}, {modules, []}, {registered, [emqx_management_sup]}, - {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]}, + {applications, [ + kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http, emqx_http_lib + ]}, {mod, {emqx_mgmt_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index e0887d788..e3930aa50 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -105,7 +105,7 @@ }. -type db_error_details() :: #{mria:table() => {error, _}}. --type config_error_details() :: #{emqx_utils_maps:config_path() => {error, _}}. +-type config_error_details() :: #{emqx_utils_maps:config_key_path() => {error, _}}. -type import_res() :: {ok, #{db_errors => db_error_details(), config_errors => config_error_details()}} | {error, _}. diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 22d18c180..301155cc0 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -86,7 +86,7 @@ -export_type([with_id_return/0, with_id_return/1]). -type state() :: #{ - publish_timer := maybe(timer:tref()), + publish_timer := maybe(reference()), publish_at := non_neg_integer(), stats_timer := maybe(reference()), stats_fun := maybe(fun((pos_integer()) -> ok)) diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 377644cdf..67c6b0ceb 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -3,7 +3,7 @@ {description, "EMQX Modules"}, {vsn, "5.0.24"}, {modules, []}, - {applications, [kernel, stdlib, emqx, emqx_ctl]}, + {applications, [kernel, stdlib, emqx, emqx_ctl, observer_cli]}, {mod, {emqx_modules_app, []}}, {registered, [emqx_modules_sup]}, {env, []} diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 37dc3c207..66fce9fde 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -83,7 +83,7 @@ server() -> %% =================================================================== callback_mode() -> always_sync. --spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +-spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl index 11c0df3fa..5d346f413 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_evacuation.erl @@ -34,7 +34,8 @@ -export_type([ start_opts/0, - start_error/0 + start_error/0, + migrate_to/0 ]). -ifdef(TEST). diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index c12086fce..cfc67aa53 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -66,7 +66,7 @@ % be sync for now. callback_mode() -> always_sync. --spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +-spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 660e95bd6..eaabb8b1c 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -100,7 +100,7 @@ adjust_fields(Fields) -> %% =================================================================== callback_mode() -> always_sync. --spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. +-spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. on_start( InstId, #{ diff --git a/apps/emqx_prometheus/src/emqx_prometheus_mria.erl b/apps/emqx_prometheus/src/emqx_prometheus_mria.erl index 91a923a2f..e08e2c405 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_mria.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_mria.erl @@ -42,7 +42,7 @@ deregister_cleanup(_) -> ok. %% @private -spec collect_mf(_Registry, Callback) -> ok when _Registry :: prometheus_registry:registry(), - Callback :: prometheus_collector:callback(). + Callback :: prometheus_collector:collect_mf_callback(). collect_mf(_Registry, Callback) -> case mria_rlog:backend() of rlog -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 37571298e..98e1a785e 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -80,8 +80,9 @@ -type queue_query() :: ?QUERY(reply_fun(), request(), HasBeenSent :: boolean(), expire_at()). -type request() :: term(). -type request_from() :: undefined | gen_statem:from(). --type request_ttl() :: infinity | timer:time(). --type health_check_interval() :: timer:time(). +-type timeout_ms() :: emqx_schema:timeout_duration_ms(). +-type request_ttl() :: emqx_schema:timeout_duration_ms(). +-type health_check_interval() :: pos_integer(). -type state() :: blocked | running. -type inflight_key() :: integer(). -type counters() :: #{ @@ -101,13 +102,13 @@ inflight_tid := inflight_table(), async_workers := #{pid() => reference()}, batch_size := pos_integer(), - batch_time := timer:time(), + batch_time := timeout_ms(), counters := counters(), - metrics_flush_interval := timer:time(), + metrics_flush_interval := timeout_ms(), queue := replayq:q(), - resume_interval := timer:time(), - tref := undefined | {timer:tref() | reference(), reference()}, - metrics_tref := undefined | {timer:tref() | reference(), reference()} + resume_interval := timeout_ms(), + tref := undefined | {reference(), reference()}, + metrics_tref := undefined | {reference(), reference()} }. callback_mode() -> [state_functions, state_enter]. @@ -2032,7 +2033,7 @@ replayq_opts(Id, Index, Opts) -> %% timeout is <= resume interval and the buffer worker is ever %% blocked, than all queued requests will basically fail without being %% attempted. --spec default_resume_interval(request_ttl(), health_check_interval()) -> timer:time(). +-spec default_resume_interval(request_ttl(), health_check_interval()) -> timeout_ms(). default_resume_interval(_RequestTTL = infinity, HealthCheckInterval) -> max(1, HealthCheckInterval); default_resume_interval(RequestTTL, HealthCheckInterval) -> diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 4d2e55681..6041dc2fb 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -47,7 +47,8 @@ fields("resource_opts") -> fields("creation_opts") -> create_opts([]). --spec create_opts([{atom(), hocon_schema:field_schema_map()}]) -> [{atom(), hocon_schema:field()}]. +-spec create_opts([{atom(), hocon_schema:field_schema()}]) -> + [{atom(), hocon_schema:field_schema()}]. create_opts(Overrides) -> override( [ diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 49831a9e8..73b7fbf90 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -307,7 +307,7 @@ clean(Context) -> Mod = get_backend_module(), Mod:clean(Context). --spec update_config(state(), hocons:config(), hocons:config()) -> state(). +-spec update_config(state(), hocon:config(), hocon:config()) -> state(). update_config(State, Conf, OldConf) -> update_config( maps:get(enable, Conf), @@ -317,7 +317,7 @@ update_config(State, Conf, OldConf) -> OldConf ). --spec update_config(boolean(), boolean(), state(), hocons:config(), hocons:config()) -> state(). +-spec update_config(boolean(), boolean(), state(), hocon:config(), hocon:config()) -> state(). update_config(false, _, State, _, _) -> disable_retainer(State); update_config(true, false, State, NewConf, _) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_index.erl b/apps/emqx_retainer/src/emqx_retainer_index.erl index d1e5425ed..ec49e7a65 100644 --- a/apps/emqx_retainer/src/emqx_retainer_index.erl +++ b/apps/emqx_retainer/src/emqx_retainer_index.erl @@ -83,7 +83,7 @@ index_score(Index, Tokens) -> %% Returns `undefined' if there are no indices with score `> 0'. %% %% @see index_score/2 --spec select_index(emqx:words(), list(index())) -> index() | undefined. +-spec select_index(emqx_types:words(), list(index())) -> index() | undefined. select_index(Tokens, Indices) -> select_index(Tokens, Indices, 0, undefined). diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index bd17dc6c4..2658dfe69 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index be91a19d2..e5454bfc9 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -82,7 +82,7 @@ update_profile(ProfileId, ProfileConfig) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_conf:update_config(ProfileId, ProfileConfig). -spec start_uploader(profile_id(), emqx_s3_uploader:opts()) -> - supervisor:start_ret() | {error, profile_not_found}. + emqx_types:startlink_ret() | {error, profile_not_found}. start_uploader(ProfileId, Opts) when ?IS_PROFILE_ID(ProfileId) -> emqx_s3_profile_uploader_sup:start_uploader(ProfileId, Opts). diff --git a/apps/emqx_s3/src/emqx_s3_client.erl b/apps/emqx_s3/src/emqx_s3_client.erl index c062cf1ca..73cf667a1 100644 --- a/apps/emqx_s3/src/emqx_s3_client.erl +++ b/apps/emqx_s3/src/emqx_s3_client.erl @@ -28,7 +28,12 @@ -export_type([ client/0, - headers/0 + headers/0, + key/0, + upload_id/0, + etag/0, + part_number/0, + config/0 ]). -type headers() :: #{binary() | string() => iodata()}. diff --git a/apps/emqx_s3/src/emqx_s3_profile_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_sup.erl index c39fc9f4b..755b78bb2 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_sup.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_sup.erl @@ -15,7 +15,7 @@ -export([init/1]). --spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> supervisor:start_ret(). +-spec start_link(emqx_s3:profile_id(), emqx_s3:profile_config()) -> emqx_types:startlink_ret(). start_link(ProfileId, ProfileConfig) -> supervisor:start_link(?MODULE, [ProfileId, ProfileConfig]). diff --git a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl index fb7b93a15..67a36a793 100644 --- a/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl +++ b/apps/emqx_s3/src/emqx_s3_profile_uploader_sup.erl @@ -24,7 +24,7 @@ -type id() :: {?MODULE, emqx_s3:profile_id()}. --spec start_link(emqx_s3:profile_id()) -> supervisor:start_ret(). +-spec start_link(emqx_s3:profile_id()) -> emqx_types:startlink_ret(). start_link(ProfileId) -> supervisor:start_link(?VIA_GPROC(id(ProfileId)), ?MODULE, [ProfileId]). @@ -44,7 +44,7 @@ id(ProfileId) -> {?MODULE, ProfileId}. -spec start_uploader(emqx_s3:profile_id(), emqx_s3_uploader:opts()) -> - supervisor:start_ret() | {error, profile_not_found}. + emqx_types:startlink_ret() | {error, profile_not_found}. start_uploader(ProfileId, Opts) -> try supervisor:start_child(?VIA_GPROC(id(ProfileId)), [Opts]) of Result -> Result diff --git a/apps/emqx_s3/src/emqx_s3_sup.erl b/apps/emqx_s3/src/emqx_s3_sup.erl index 0f6b0160b..1a8618e62 100644 --- a/apps/emqx_s3/src/emqx_s3_sup.erl +++ b/apps/emqx_s3/src/emqx_s3_sup.erl @@ -16,7 +16,7 @@ -export([init/1]). --spec start_link() -> supervisor:start_ret(). +-spec start_link() -> emqx_types:startlink_ret(). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). diff --git a/apps/emqx_s3/src/emqx_s3_uploader.erl b/apps/emqx_s3/src/emqx_s3_uploader.erl index aa547c7cc..548eaf8c6 100644 --- a/apps/emqx_s3/src/emqx_s3_uploader.erl +++ b/apps/emqx_s3/src/emqx_s3_uploader.erl @@ -33,7 +33,12 @@ format_status/2 ]). --export_type([opts/0]). +-export_type([opts/0, config/0]). + +-type config() :: #{ + min_part_size => pos_integer(), + max_part_size => pos_integer() +}. -type opts() :: #{ key := string(), diff --git a/apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl b/apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl index 06c96638c..9e36afc29 100644 --- a/apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl +++ b/apps/emqx_utils/src/bpapi/emqx_bpapi_trans.erl @@ -191,7 +191,7 @@ push_err(Line, Err, S = #s{errors = Errs}) -> push_target(Target, S = #s{targets = Targets}) -> S#s{targets = [Target | Targets]}. --spec api_and_version(module()) -> {ok, emqx_bpapi:api(), emqx_bpapi:version()} | error. +-spec api_and_version(module()) -> {ok, emqx_bpapi:api(), emqx_bpapi:api_version()} | error. api_and_version(Module) -> Opts = [{capture, all_but_first, list}], case re:run(atom_to_list(Module), "(.*)_proto_v([0-9]+)$", Opts) of diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index b5bf68dff..1d43cc56c 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -79,6 +79,12 @@ -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). +-export_type([ + readable_error_msg/1 +]). + +-type readable_error_msg(_Error) :: binary(). + -type maybe(T) :: undefined | T. -dialyzer({nowarn_function, [nolink_apply/2]}). @@ -435,7 +441,7 @@ pmap(Fun, List, Timeout) when nolink_apply(Fun) -> nolink_apply(Fun, infinity). %% @doc Same as `nolink_apply/1', with a timeout. --spec nolink_apply(function(), timer:timeout()) -> term(). +-spec nolink_apply(function(), timeout()) -> term(). nolink_apply(Fun, Timeout) when is_function(Fun, 0) -> Caller = self(), ResRef = alias([reply]), From 6b553e37c07a80ed6b52aa94a962e54f9ddc2a06 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 15:01:40 +0100 Subject: [PATCH 40/46] chore: pin ekka 0.17.0 previously 0.15.16 change summary: 1. otp 26 2. type spec fixes 3. dropped UDP mcast clustering --- apps/emqx/rebar.config | 2 +- apps/emqx_machine/test/emqx_machine_SUITE.erl | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 27c7500d7..62d7cc0b5 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -28,7 +28,7 @@ {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.17.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, diff --git a/apps/emqx_machine/test/emqx_machine_SUITE.erl b/apps/emqx_machine/test/emqx_machine_SUITE.erl index 78e95d083..7b64df05c 100644 --- a/apps/emqx_machine/test/emqx_machine_SUITE.erl +++ b/apps/emqx_machine/test/emqx_machine_SUITE.erl @@ -93,7 +93,7 @@ t_custom_shard_transports(_Config) -> Shard = test_shard, %% the config keys are binaries ShardBin = atom_to_binary(Shard), - DefaultTransport = gen_rpc, + DefaultTransport = distr, ?assertEqual(DefaultTransport, mria_config:shard_transport(Shard)), application:set_env(emqx_machine, custom_shard_transports, #{ShardBin => distr}), emqx_machine:start(), diff --git a/mix.exs b/mix.exs index 838e2c86b..598cb3b2d 100644 --- a/mix.exs +++ b/mix.exs @@ -55,7 +55,7 @@ defmodule EMQXUmbrella.MixProject do {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.9", override: true}, {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.17.0", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.0", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.15", override: true}, diff --git a/rebar.config b/rebar.config index 2efcb0e3b..d1e88b482 100644 --- a/rebar.config +++ b/rebar.config @@ -71,7 +71,7 @@ , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}} , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-2"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.17.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.15"}}} From 9a6056ff2fb36b71a692a01e68e5feb2185fec48 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 15:18:51 +0100 Subject: [PATCH 41/46] refactor(emqx_machine): move system_monitor to included_applications system_monitor has to be found in application (or included_applications) otherwise dialyzer fails with Unknown function Adding to 'applications' would cause a deadlock because one cannot call application:ensure_all_started in the application/2 callback. --- apps/emqx_machine/priv/reboot_lists.eterm | 18 +++++++++--------- apps/emqx_machine/src/emqx_machine.app.src | 8 +++++++- apps/emqx_machine/src/emqx_machine.erl | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 5 +---- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 6dffd5e2d..cf3ad1523 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -24,8 +24,15 @@ runtime_tools, redbug, xmerl, + %% has no application/2 callback {hocon, load}, - telemetry + telemetry, + observer_cli, + covertool, + tools, + observer, + {system_monitor, load}, + jq ], %% must always be of type `load' common_business_apps => @@ -33,11 +40,6 @@ emqx, emqx_conf, esasl, - observer_cli, - tools, - covertool, - %% started by emqx_machine - system_monitor, emqx_utils, emqx_durable_storage, emqx_http_lib, @@ -79,9 +81,7 @@ emqx_plugins, emqx_opentelemetry, quicer, - bcrypt, - jq, - observer + bcrypt ], %% must always be of type `load' ee_business_apps => diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 904d5fe77..e3b37fc23 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -6,7 +6,13 @@ {vsn, "0.2.17"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib, emqx_ctl, system_monitor, covertool]}, + {applications, [kernel, stdlib, emqx_ctl, covertool]}, + %% system_monitor is loaded but not booted, + %% emqx_machine.erl makes the decision when to start + %% the app after certain config injection. + %% it's a included_application because otherwise dialyzer + %% would report unknown functions + {included_applications, [system_monitor]}, {mod, {emqx_machine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 8dc385fb3..fee7fa7aa 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -47,7 +47,7 @@ start() -> os:set_signal(sigterm, handle) end, ok = set_backtrace_depth(), - start_sysmon(), + ok = start_sysmon(), configure_shard_transports(), set_mnesia_extra_diagnostic_checks(), emqx_otel_app:configure_otel_deps(), diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index afb195543..08cf8c448 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -36,9 +36,6 @@ %% If any of these applications crash, the entire EMQX node shuts down: -define(BASIC_PERMANENT_APPS, [mria, ekka, esockd, emqx]). -%% These apps should NOT be (re)started automatically: --define(EXCLUDED_APPS, [system_monitor, observer_cli, jq]). - %% These apps are optional, they may or may not be present in the %% release, depending on the build flags: -define(OPTIONAL_APPS, [bcrypt, observer]). @@ -157,7 +154,7 @@ basic_reboot_apps() -> excluded_apps() -> %% Optional apps _should_ be (re)started automatically, but only %% when they are found in the release: - ?EXCLUDED_APPS ++ [App || App <- ?OPTIONAL_APPS, not is_app(App)]. + [App || App <- ?OPTIONAL_APPS, not is_app(App)]. is_app(Name) -> case application:load(Name) of From 33a7282cdd6d80a52435048b94e725dcb1059fb5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 15:31:01 +0100 Subject: [PATCH 42/46] fix(dialyzer): only include eunit when TEST is defined --- apps/emqx_auth/src/emqx_auth.app.src | 2 +- .../src/emqx_authn/emqx_authn_schema.erl | 2 ++ .../src/schema/emqx_bridge_v2_schema.erl | 3 +++ .../src/schema/emqx_connector_schema.erl | 15 +++++++++------ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/emqx_auth/src/emqx_auth.app.src b/apps/emqx_auth/src/emqx_auth.app.src index 3d9109fd1..587c4b239 100644 --- a/apps/emqx_auth/src/emqx_auth.app.src +++ b/apps/emqx_auth/src/emqx_auth.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth, [ {description, "EMQX Authentication and authorization"}, - {vsn, "0.1.28"}, + {vsn, "0.1.29"}, {modules, []}, {registered, [emqx_auth_sup]}, {applications, [ diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index 371c6f2be..e1ecad42b 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -21,7 +21,9 @@ -include("emqx_authn.hrl"). -include("emqx_authn_chains.hrl"). +-ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-endif. -behaviour(emqx_schema_hooks). -export([ diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 2c2dde4da..7cd6b5bd8 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -18,7 +18,10 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). + +-ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-endif. -import(hoconsc, [mk/2, ref/2]). diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 31e3a335f..39ae3e764 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -18,7 +18,10 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). + +-ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +-endif. -import(hoconsc, [mk/2, ref/2]). @@ -557,6 +560,12 @@ to_bin(Bin) when is_binary(Bin) -> to_bin(Something) -> Something. +node_name() -> + {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. + +status() -> + hoconsc:enum([connected, disconnected, connecting, inconsistent]). + -ifdef(TEST). -include_lib("hocon/include/hocon_types.hrl"). schema_homogeneous_test() -> @@ -591,12 +600,6 @@ is_bad_schema(#{type := ?MAP(_, ?R_REF(Module, TypeName))}) -> }} end. -status() -> - hoconsc:enum([connected, disconnected, connecting, inconsistent]). - -node_name() -> - {"node", mk(binary(), #{desc => ?DESC("desc_node_name"), example => "emqx@127.0.0.1"})}. - common_field_names() -> [ enable, description From 7560016f0ef7e6276d4befd51697d1b48d2fb1b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 6 Dec 2023 19:23:53 +0100 Subject: [PATCH 43/46] fix(hocon): upgrade hocon from 0.40.0 to 0.40.1 fixed type spec, exported some public types from emqx_schema module --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 62d7cc0b5..bcc007141 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -30,7 +30,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.9"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.17.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.0"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.1"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index 598cb3b2d..d7aa8885f 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.40.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.40.1", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index d1e88b482..fffbf202b 100644 --- a/rebar.config +++ b/rebar.config @@ -84,7 +84,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From c45ed7b8880ad55b5178303af79866726d4e0eab Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 7 Dec 2023 09:44:48 -0300 Subject: [PATCH 44/46] fix(connectors_api): fill default raw config values before returning Fixes https://emqx.atlassian.net/browse/EMQX-11562 --- apps/emqx_connector/src/emqx_connector_api.erl | 17 ++++++++++++++++- .../test/emqx_connector_api_SUITE.erl | 13 +++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index f25fe9b7e..af5721585 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -610,11 +610,12 @@ format_resource( #{ type := Type, name := ConnectorName, - raw_config := RawConf, + raw_config := RawConf0, resource_data := ResourceData }, Node ) -> + RawConf = fill_defaults(Type, RawConf0), redact( maps:merge( RawConf#{ @@ -638,6 +639,20 @@ format_resource_data(added_channels, Channels, Result) -> format_resource_data(K, V, Result) -> Result#{K => V}. +fill_defaults(Type, RawConf) -> + PackedConf = pack_connector_conf(Type, RawConf), + FullConf = emqx_config:fill_defaults(emqx_connector_schema, PackedConf, #{}), + unpack_connector_conf(Type, FullConf). + +pack_connector_conf(Type, RawConf) -> + #{<<"connectors">> => #{bin(Type) => #{<<"foo">> => RawConf}}}. + +unpack_connector_conf(Type, PackedConf) -> + TypeBin = bin(Type), + #{<<"connectors">> := Bridges} = PackedConf, + #{<<"foo">> := RawConf} = maps:get(TypeBin, Bridges), + RawConf. + format_action(ActionId) -> element(2, emqx_bridge_v2:parse_id(ActionId)). diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index c020c2504..92fd1de6d 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -845,6 +845,19 @@ t_fail_delete_with_action(Config) -> ), ok. +t_raw_config_response_defaults(Config) -> + Params = maps:without([<<"enable">>, <<"resource_opts">>], ?KAFKA_CONNECTOR(?CONNECTOR_NAME)), + ?assertMatch( + {ok, 201, #{<<"enable">> := true, <<"resource_opts">> := #{}}}, + request_json( + post, + uri(["connectors"]), + Params, + Config + ) + ), + ok. + %%% helpers listen_on_random_port() -> SockOpts = [binary, {active, false}, {packet, raw}, {reuseaddr, true}, {backlog, 1000}], From 583645d97bc6e0eb6b0c84dd966b1ccc3d556836 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 7 Dec 2023 07:51:04 +0100 Subject: [PATCH 45/46] chore: bump emqtt 1.10.1 and quicer 0.0.308 --- apps/emqx/rebar.config.script | 2 +- mix.exs | 4 ++-- rebar.config | 2 +- rebar.config.erl | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index d68998c4c..8f9de344c 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,7 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.303"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.308"}}}. Dialyzer = fun(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), diff --git a/mix.exs b/mix.exs index d7aa8885f..21a07dee4 100644 --- a/mix.exs +++ b/mix.exs @@ -64,7 +64,7 @@ defmodule EMQXUmbrella.MixProject do {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, # maybe forbid to fetch quicer {:emqtt, - github: "emqx/emqtt", tag: "1.10.0", override: true, system_env: maybe_no_quic_env()}, + github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()}, {:rulesql, github: "emqx/rulesql", tag: "0.1.7"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, @@ -800,7 +800,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.303", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.308", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index fffbf202b..830ffc051 100644 --- a/rebar.config +++ b/rebar.config @@ -78,7 +78,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.0"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.10.1"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.7"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 37f041bf6..7ab693979 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.1"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.303"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.308"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.12"}}}. From 499f4d4bc5926245fbdee0b448c133dd9354e722 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 7 Dec 2023 10:10:04 +0100 Subject: [PATCH 46/46] chore(quicer): dialyzer --- apps/emqx/src/emqx_quic_connection.erl | 6 ++++-- apps/emqx/src/emqx_quic_data_stream.erl | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index e081dc5cf..b75137e05 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -47,13 +47,15 @@ handle_info/2 ]). +-export_type([cb_state/0, cb_ret/0]). + -type cb_state() :: #{ %% connecion owner pid conn_pid := pid(), %% Pid of ctrl stream ctrl_pid := undefined | pid(), %% quic connecion handle - conn := undefined | quicer:conneciton_handle(), + conn := undefined | quicer:connection_handle(), %% Data streams that handoff from this process %% these streams could die/close without effecting the connecion/session. %@TODO type? @@ -85,7 +87,7 @@ init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> init(ConnOpts) when is_map(ConnOpts) -> {ok, init_cb_state(ConnOpts)}. --spec closed(quicer:conneciton_handle(), quicer:conn_closed_props(), cb_state()) -> +-spec closed(quicer:connection_handle(), quicer:conn_closed_props(), cb_state()) -> {stop, normal, cb_state()}. closed(_Conn, #{is_peer_acked := _} = Prop, S) -> ?SLOG(debug, Prop), diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl index 254e6e2a3..48d8ddd2e 100644 --- a/apps/emqx/src/emqx_quic_data_stream.erl +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -124,7 +124,7 @@ send_complete(_Stream, false, S) -> send_complete(_Stream, true = _IsCanceled, S) -> {ok, S}. --spec send_shutdown_complete(stream_handle(), error_code(), cb_state()) -> cb_ret(). +-spec send_shutdown_complete(stream_handle(), IsGraceful :: boolean(), cb_state()) -> cb_ret(). send_shutdown_complete(_Stream, _Flags, S) -> {ok, S}. @@ -321,7 +321,7 @@ serialize_packet(Packet, Serialize) -> -spec init_state( quicer:stream_handle(), quicer:connection_handle(), - quicer:new_stream_props() + non_neg_integer() ) -> % @TODO map().