From 1bb31776c944385a718b877f5711adf9ae3c53e1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 14 Nov 2022 09:38:18 +0800 Subject: [PATCH 01/27] fix(rewrite): add topic rewrite validation for filter or name --- changes/v4.3.23-en.md | 7 ++ changes/v4.3.23-zh.md | 7 ++ lib-ce/emqx_modules/src/emqx_mod_rewrite.erl | 90 ++++++++++++++----- .../test/emqx_mod_rewrite_SUITE.erl | 2 +- 4 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 changes/v4.3.23-en.md create mode 100644 changes/v4.3.23-zh.md diff --git a/changes/v4.3.23-en.md b/changes/v4.3.23-en.md new file mode 100644 index 000000000..d36322211 --- /dev/null +++ b/changes/v4.3.23-en.md @@ -0,0 +1,7 @@ +# v4.3.23 + +## Enhancements + +- Added topic validation for `emqx_mod_rewrite`. The dest topics contains wildcards are not allowed to publish [#9359](https://github.com/emqx/emqx/issues/9359). + +## Bug fixes diff --git a/changes/v4.3.23-zh.md b/changes/v4.3.23-zh.md new file mode 100644 index 000000000..e290b9ad3 --- /dev/null +++ b/changes/v4.3.23-zh.md @@ -0,0 +1,7 @@ +# v4.3.23 + +## 增强 + +- 为主题重写模块增加主题合法性检查,带有通配符的目标主题不允许被发布 [#9359](https://github.com/emqx/emqx/issues/9359)。 + +## 修复 diff --git a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl index 9f0f635d9..562f0a916 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl @@ -23,7 +23,7 @@ -include_lib("emqx/include/logger.hrl"). -ifdef(TEST). --export([ compile/1 +-export([ compile_rules/1 , match_and_rewrite/3 ]). -endif. @@ -40,13 +40,14 @@ , description/0 ]). +-type(topic() :: binary()). + %%-------------------------------------------------------------------- %% Load/Unload %%-------------------------------------------------------------------- load(RawRules) -> - {PubRules, SubRules} = compile(RawRules), - log_start(RawRules), + {PubRules, SubRules} = compile_rules(RawRules), emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}, 1000), emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}, 1000), emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}, 1000). @@ -75,30 +76,59 @@ description() -> %% Internal functions %%-------------------------------------------------------------------- -log_start(Rules) -> - PubRules = [{pub, Topic, Re, Dest} || {rewrite, pub, Topic, Re, Dest} <- Rules], - SubRules = [{sub, Topic, Re, Dest} || {rewrite, sub, Topic, Re, Dest} <- Rules], +compile_rules(RawRules) -> + compile(validate_rules(RawRules)). + +compile({PubRules, SubRules}) -> + CompileRE = + fun({rewrite, RewriteFrom, Re, RewriteTo}) -> + {ok, MP} = re:compile(Re), + {rewrite, RewriteFrom, MP, RewriteTo} + end, + {lists:map(CompileRE, PubRules), lists:map(CompileRE, SubRules)}. + +validate_rules(Rules) -> + PubRules = [{rewrite, RewriteFrom, Re, RewriteTo} || + {rewrite, pub, RewriteFrom, Re, RewriteTo} <- Rules, + validate_rule(pub, RewriteFrom, RewriteTo) + ], + SubRules = [{rewrite, RewriteFrom, Re, RewriteTo} || + {rewrite, sub, RewriteFrom, Re, RewriteTo} <- Rules, + validate_rule(sub, RewriteFrom, RewriteTo) + ], ?LOG(info, "[Rewrite] Load: pub rules count ~p sub rules count ~p", [erlang:length(PubRules), erlang:length(SubRules)]), - log_rule(PubRules, 1), - log_rule(SubRules, 1). + log_rules(pub, PubRules), + log_rules(sub, SubRules), + {PubRules, SubRules}. -log_rule([], _Index) -> ok; -log_rule([{Type, Topic, Re, Dest} | Rules], Index) -> +validate_rule(Type, RewriteFrom, RewriteTo) -> + case validate_topic(filter, RewriteFrom) of + ok -> + case validate_topic(dest_topic_type(Type), RewriteTo) of + ok -> + true; + {error, Reason} -> + log_invalid_rule(to, Type, RewriteTo, Reason), + false + end; + {error, Reason} -> + log_invalid_rule(from, Type, RewriteFrom, Reason), + false + end. + +log_invalid_rule(Direction, Type, Topic, Reason) -> + ?LOG(warning, "Invalid rewrite ~p rule for rewrite ~p topic '~ts' discarded. Reason: ~p", + [type_to_name(Type), Direction, Topic, Reason]). + +log_rules(Type, Rules) -> + do_log_rules(Type, Rules, 1). + +do_log_rules(_Type, [], _Index) -> ok; +do_log_rules(Type, [{_, Topic, Re, Dest} | Rules], Index) -> ?LOG(info, "[Rewrite] Load ~p rule[~p]: source: ~ts, re: ~ts, dest: ~ts", [Type, Index, Topic, Re, Dest]), - log_rule(Rules, Index + 1). - -compile(Rules) -> - PubRules = [ begin - {ok, MP} = re:compile(Re), - {rewrite, Topic, MP, Dest} - end || {rewrite, pub, Topic, Re, Dest}<- Rules ], - SubRules = [ begin - {ok, MP} = re:compile(Re), - {rewrite, Topic, MP, Dest} - end || {rewrite, sub, Topic, Re, Dest}<- Rules ], - {PubRules, SubRules}. + do_log_rules(Type, Rules, Index + 1). match_and_rewrite(Topic, [], _) -> Topic; @@ -138,3 +168,19 @@ filter_client_binds(Binds) -> (_) -> true end, Binds). + +type_to_name(pub) -> 'PUBLISH'; +type_to_name(sub) -> 'SUBSCRIBE'. + +dest_topic_type(pub) -> name; +dest_topic_type(sub) -> filter. + +-spec(validate_topic(name | filter, topic()) -> ok | {error, term()}). +validate_topic(Type, Topic) -> + try + true = emqx_topic:validate(Type, Topic), + ok + catch + error:Reason -> + {error, Reason} + end. diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl index 44574344b..16c9b6188 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl @@ -82,7 +82,7 @@ t_mod_rewrite(_Config) -> ok = emqx_mod_rewrite:unload(?RULES). t_rewrite_rule(_Config) -> - {PubRules, SubRules} = emqx_mod_rewrite:compile(?RULES), + {PubRules, SubRules} = emqx_mod_rewrite:compile_rules(?RULES), ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules, [])), ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules, [])), ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules, [])), From 32a885017b17597cdecef486e851d089cc26ce09 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 14 Nov 2022 16:19:05 +0800 Subject: [PATCH 02/27] chore: bump appup.src for `emqx_mod_rewrite` --- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.appup.src | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index 0c3328f8c..68b210d26 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.10"}, + {vsn, "4.3.11"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 461eefa15..47d9c9a75 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,9 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.3.10",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.3.9", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-7]">>, [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, @@ -35,9 +39,13 @@ {load_module,emqx_mod_api_topic_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.9",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.3.10",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.3.9", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_modules,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_modules,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[5-7]">>, [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, From db08efa0acce527b0a85490cdb27a0e1e324c2fb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 14 Nov 2022 16:40:01 +0800 Subject: [PATCH 03/27] test(rewrite): invalided topic rewrite rule --- .../test/emqx_mod_rewrite_SUITE.erl | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl index 16c9b6188..5165f9e9d 100644 --- a/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl +++ b/lib-ce/emqx_modules/test/emqx_mod_rewrite_SUITE.erl @@ -30,6 +30,13 @@ {rewrite, sub, <<"c/#">>,<<"^c/(.+)$">>,<<"sub/%c/$1">>} ]). +-define(BAD_RULES_1, [{rewrite, pub, <<"x/#">>,<<"^x/y/(.+)$">>,<<"z/y/+">>}]). + +%% empty topic filter/name won't be ranched cased `emqx.conf` will be checked before emqx started +%% but we need this check for emqx-ee modules api +-define(BAD_RULES_2, [{rewrite, pub, <<"">>,<<"^x/y/(.+)$">>,<<"z/y/+">>}]). +-define(BAD_RULES_3, [{rewrite, pub, <<"name/#">>,<<"^name/(.+)$">>,<<"">>}]). + all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> @@ -83,11 +90,29 @@ t_mod_rewrite(_Config) -> t_rewrite_rule(_Config) -> {PubRules, SubRules} = emqx_mod_rewrite:compile_rules(?RULES), + %% assert ordering + ?assertMatch([{rewrite, <<"x/#">>, _, <<"z/y/$1">>}, + {rewrite, <<"name/#">>, _, <<"pub/%u/$1">>}, + {rewrite, <<"c/#">>, _, <<"pub/%c/$1">>}], + PubRules), + ?assertMatch([{rewrite, <<"y/+/z/#">>, _, <<"y/z/$2">>}, + {rewrite, <<"name/#">>, _, <<"sub/%u/$1">>}, + {rewrite, <<"c/#">>, _, <<"sub/%c/$1">>}], + SubRules), + ?assertEqual(<<"z/y/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/y/2">>, PubRules, [])), ?assertEqual(<<"x/1/2">>, emqx_mod_rewrite:match_and_rewrite(<<"x/1/2">>, PubRules, [])), ?assertEqual(<<"y/z/b">>, emqx_mod_rewrite:match_and_rewrite(<<"y/a/z/b">>, SubRules, [])), ?assertEqual(<<"y/def">>, emqx_mod_rewrite:match_and_rewrite(<<"y/def">>, SubRules, [])). +t_rewrite_bad_rule_1(_Config) -> + ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_1)). + +t_rewrite_bad_rule_2(_Config) -> + ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_2)). + +t_rewrite_bad_rule_3(_Config) -> + ?assertEqual({[], []}, emqx_mod_rewrite:compile_rules(?BAD_RULES_3)). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- From ebfaecfa915fe43c813a7e8a88f7e4c32b0a9657 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 10 Nov 2022 13:16:02 +0800 Subject: [PATCH 04/27] fix(jwt): fix the jwt ACL will return a wrong result when the token is expired --- apps/emqx_auth_jwt/src/emqx_auth_jwt.erl | 2 +- apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index 623fa25c5..223fc673c 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -67,7 +67,7 @@ check_acl(ClientInfo = #{jwt_claims := Claims}, case is_expired(Exp) of true -> ?DEBUG("acl_deny_due_to_jwt_expired", []), - deny; + {stop, deny}; false -> verify_acl(ClientInfo, Acl, PubSub, Topic) end; diff --git a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl index 7452091bd..235f77783 100644 --- a/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl +++ b/apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl @@ -462,6 +462,16 @@ t_check_jwt_acl_expire(_Config) -> {ok, #{}, [?RC_NOT_AUTHORIZED]}, emqtt:subscribe(C, <<"a/b">>, 0)), + Default = emqx_zone:get_env(external, acl_nomatch, deny), + emqx_zone:set_env(external, acl_nomatch, allow), + try + ?assertMatch( + {ok, #{}, [?RC_NOT_AUTHORIZED]}, + emqtt:subscribe(C, <<"a/b">>, 0)) + after + emqx_zone:set_env(external, acl_nomatch, Default) + end, + ok = emqtt:disconnect(C). t_check_jwt_acl_no_exp(init, _Config) -> From 8a0158e21dd67cc4c2d03e8dd82e21ee85798895 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 15 Nov 2022 09:52:34 +0800 Subject: [PATCH 05/27] chore: update changes --- changes/v4.3.23-en.md | 2 ++ changes/v4.3.23-zh.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changes/v4.3.23-en.md b/changes/v4.3.23-en.md index d36322211..ee43c4cee 100644 --- a/changes/v4.3.23-en.md +++ b/changes/v4.3.23-en.md @@ -5,3 +5,5 @@ - Added topic validation for `emqx_mod_rewrite`. The dest topics contains wildcards are not allowed to publish [#9359](https://github.com/emqx/emqx/issues/9359). ## Bug fixes + +- Fix a bug where the JWT ACL would not short-circuit with a deny response when the token is expired [#9338](https://github.com/emqx/emqx/pull/9338). diff --git a/changes/v4.3.23-zh.md b/changes/v4.3.23-zh.md index e290b9ad3..3e3e293f4 100644 --- a/changes/v4.3.23-zh.md +++ b/changes/v4.3.23-zh.md @@ -5,3 +5,5 @@ - 为主题重写模块增加主题合法性检查,带有通配符的目标主题不允许被发布 [#9359](https://github.com/emqx/emqx/issues/9359)。 ## 修复 + +- 修复 JWT ACL 在令牌超期后授权检查不生效的问题 [#9338](https://github.com/emqx/emqx/pull/9338)。 From f4a14ecf66cf1f9c2389d37ed6aa964d308f91ed Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 10 Nov 2022 15:08:36 +0800 Subject: [PATCH 06/27] refactor(cookie): Warning message when boot with default Erlang cookie --- bin/emqx | 8 ++++++-- data/emqx_vars | 4 ++++ rebar.config.erl | 19 ++++++++++++------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/bin/emqx b/bin/emqx index 89fc564b1..cdbb7c9de 100755 --- a/bin/emqx +++ b/bin/emqx @@ -537,9 +537,13 @@ if [ -z "$COOKIE" ]; then fi fi -if [ -z "$COOKIE" ]; then +[ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE" +if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then + echoerr "!!!!!!" + echoerr "WARNING: Default (insecure) Erlang cookie is in use." echoerr "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE_COOKIE" - exit 1 + echoerr "NOTE: Use the same config value for all nodes in the cluster." + echoerr "!!!!!!" fi cd "$ROOTDIR" diff --git a/data/emqx_vars b/data/emqx_vars index 8ca6bf22d..1559e890c 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -14,6 +14,10 @@ RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" EMQX_DESCRIPTION='{{ emqx_description }}' +## Do not change EMQX_DEFAULT_ERLANG_COOKIE. +## Configure EMQX_NODE_COOKIE instead +EMQX_DEFAULT_ERLANG_COOKIE='{{ emqx_default_erlang_cookie }}' + ## Warning: DO NOT create new variables using the above vars in this file, ## as the vars above can be overwritten by the relup scripts later, like: ## REL_VSN="new_version" diff --git a/rebar.config.erl b/rebar.config.erl index 1f7731462..1267da0bb 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -203,18 +203,23 @@ overlay_vars(RelType, PkgType, false) -> overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType). %% vars per release type, cloud or edge -overlay_vars_rel(RelType) -> - VmArgs = case RelType of - cloud -> "vm.args"; - edge -> "vm.args.edge" - end, - [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} +overlay_vars_rel(cloud) -> + [ {vm_args_file, "vm.args"} + | overlay_vars_rel_common(cloud) + ]; +overlay_vars_rel(edge) -> + [ {vm_args_file, "vm.args.edge"} + | overlay_vars_rel_common(edge) + ]. + +overlay_vars_rel_common(RelType) -> + [ {emqx_default_erlang_cookie, "emqxsecretcookie"} + , {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} , {enable_plugin_emqx_telemetry, true} - , {vm_args_file, VmArgs} ]. %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) From ed3dc0b614274ddb2a6878af38b2cc590d8a61b6 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 10 Nov 2022 15:25:19 +0800 Subject: [PATCH 07/27] chore: update changes --- changes/v4.3.23-en.md | 2 ++ changes/v4.3.23-zh.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changes/v4.3.23-en.md b/changes/v4.3.23-en.md index d36322211..b8c0230c6 100644 --- a/changes/v4.3.23-en.md +++ b/changes/v4.3.23-en.md @@ -4,4 +4,6 @@ - Added topic validation for `emqx_mod_rewrite`. The dest topics contains wildcards are not allowed to publish [#9359](https://github.com/emqx/emqx/issues/9359). +- Print a warning message when boot with the default (insecure) Erlang cookie [#9340](https://github.com/emqx/emqx/pull/9340). + ## Bug fixes diff --git a/changes/v4.3.23-zh.md b/changes/v4.3.23-zh.md index e290b9ad3..b14dd96bb 100644 --- a/changes/v4.3.23-zh.md +++ b/changes/v4.3.23-zh.md @@ -4,4 +4,6 @@ - 为主题重写模块增加主题合法性检查,带有通配符的目标主题不允许被发布 [#9359](https://github.com/emqx/emqx/issues/9359)。 +- 使用默认的(不安全的) Erlang cookie 进行启动时,将会打印一条警告信息 [#9340](https://github.com/emqx/emqx/pull/9340)。 + ## 修复 From d53eed9e2e7a08facfd50793d8c2b84613f722ae Mon Sep 17 00:00:00 2001 From: raoxiaoli Date: Thu, 17 Nov 2022 18:34:38 +0800 Subject: [PATCH 08/27] fix: helm chart headless service name error --- changes/v4.4.11-en.md | 2 ++ changes/v4.4.11-zh.md | 2 ++ .../charts/emqx/templates/configmap.env.yaml | 13 +++++++++++++ deploy/charts/emqx/values.yaml | 19 ------------------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/changes/v4.4.11-en.md b/changes/v4.4.11-en.md index 68bf692ed..e899b12c2 100644 --- a/changes/v4.4.11-en.md +++ b/changes/v4.4.11-en.md @@ -27,3 +27,5 @@ - Fix get trace list crash when trace not initialize. [#9156](https://github.com/emqx/emqx/pull/9156) - Fix create trace sometime failed by end_at time has already passed. [#9156](https://github.com/emqx/emqx/pull/9156) + +- Fix the `Discovery error: no such service` error that occurred during helm chart deployment, resulting in an abnormal discovery of cluster nodes. [#9388](https://github.com/emqx/emqx/pull/9388) diff --git a/changes/v4.4.11-zh.md b/changes/v4.4.11-zh.md index d489a68a1..7142dd5de 100644 --- a/changes/v4.4.11-zh.md +++ b/changes/v4.4.11-zh.md @@ -24,3 +24,5 @@ - 修复日志追踪模块没开启时,GET Trace 列表接口报错的问题。[#9156](https://github.com/emqx/emqx/pull/9156) - 修复创建追踪日志时偶尔会报`end_at time has already passed`错误,导致创建失败。[#9156](https://github.com/emqx/emqx/pull/9156) + +- 修复 helm chart 部署时出现`Discovery error: no such service`错误,导致集群节点发现异常。[#9388](https://github.com/emqx/emqx/pull/9388) diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml index 1167b7972..c1abb4fdb 100644 --- a/deploy/charts/emqx/templates/configmap.env.yaml +++ b/deploy/charts/emqx/templates/configmap.env.yaml @@ -10,6 +10,19 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: + EMQX_NAME: {{ .Release.Name }} + {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY) "k8s" }} + EMQX_CLUSTER__K8S__APP_NAME: {{ .Release.Name }} + EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: {{ include "emqx.fullname" . }}-headless + EMQX_CLUSTER__K8S__NAMESPACE: {{ .Release.Namespace }} + EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname" + EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local" + {{- else if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY) "dns" }} + EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + EMQX_CLUSTER__DNS__APP: {{ .Release.Name }} + EMQX_CLUSTER__DNS__TYPE: "srv" + {{- end -}} {{- range $index, $value := .Values.emqxConfig }} {{- if $value }} {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index f9b15cacf..e69d42d97 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -72,27 +72,8 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: - EMQX_NAME: "{{ .Release.Name }}" - - ## Cluster discovery by dns - # EMQX_CLUSTER__DISCOVERY: "dns" - # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" - # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}" - # EMQX_CLUSTER__DNS__TYPE: "srv" - ## Cluster discovery by k8s EMQX_CLUSTER__DISCOVERY: "k8s" - EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}" - EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" - EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" - EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" - ## The address type is used to extract host from k8s service. - ## Value: ip | dns | hostname - ## Note:Hostname is only supported after v4.0-rc.2 - EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname" - EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local" - ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns - # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local" EMQX_MANAGEMENT__DEFAULT_APPLICATION__ID: "admin" EMQX_MANAGEMENT__DEFAULT_APPLICATION__SECRET: "public" From 61517f8280844c406ab37638eaae2435bb41d350 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 18 Nov 2022 14:23:49 +0800 Subject: [PATCH 09/27] chore: make remsh node name away from the atom table overflow --- bin/emqx | 2 +- bin/nodetool | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index cdbb7c9de..04e6b201a 100755 --- a/bin/emqx +++ b/bin/emqx @@ -319,7 +319,7 @@ relx_rem_sh() { # Generate a random id relx_gen_id() { - od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}' + od -t u -N 4 /dev/urandom | head -n1 | awk '{print $2 % 1000}' } # Control a node diff --git a/bin/nodetool b/bin/nodetool index c3bfe5b3c..234dabb97 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -202,7 +202,12 @@ nodename(Name) -> this_node_name(Name) -> [Node, Host] = re:split(Name, "@", [{return, list}, unicode]), - list_to_atom(lists:concat(["remsh_maint_", Node, os:getpid(), "@", Host])). + list_to_atom(lists:concat(["remsh_maint_", Node, node_name_suffix_id(), "@", Host])). + +%% use the reversed value that from pid mod 1000 as the node name suffix +node_name_suffix_id() -> + Pid = os:getpid(), + string:slice(string:reverse(Pid), 0, 3). %% For windows??? create_mnesia_dir(DataDir, NodeName) -> From 2cdc953435fba8f3f72a82e23b788232925a9b28 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 18 Nov 2022 14:29:29 +0800 Subject: [PATCH 10/27] chore: update changes --- changes/v4.3.23-en.md | 2 ++ changes/v4.3.23-zh.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/changes/v4.3.23-en.md b/changes/v4.3.23-en.md index fbc662fc2..c3d6216ed 100644 --- a/changes/v4.3.23-en.md +++ b/changes/v4.3.23-en.md @@ -6,6 +6,8 @@ - Print a warning message when boot with the default (insecure) Erlang cookie [#9340](https://github.com/emqx/emqx/pull/9340). +- Improve node name generation rules to avoid potential atom table overflow risk [#9391](https://github.com/emqx/emqx/pull/9391). + ## Bug fixes - Fix a bug where the JWT ACL would not short-circuit with a deny response when the token is expired [#9338](https://github.com/emqx/emqx/pull/9338). diff --git a/changes/v4.3.23-zh.md b/changes/v4.3.23-zh.md index 14f9c33e1..4f486f186 100644 --- a/changes/v4.3.23-zh.md +++ b/changes/v4.3.23-zh.md @@ -6,6 +6,8 @@ - 使用默认的(不安全的) Erlang cookie 进行启动时,将会打印一条警告信息 [#9340](https://github.com/emqx/emqx/pull/9340)。 +- 改进了节点名称生成规则,以避免潜在的原子表溢出风险 [#9391](https://github.com/emqx/emqx/pull/9391)。 + ## 修复 - 修复 JWT ACL 在令牌超期后授权检查不生效的问题 [#9338](https://github.com/emqx/emqx/pull/9338)。 From cc03bc788161627c1e2f964b6f9297af8dd57f6f Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 22 Nov 2022 16:55:47 +0800 Subject: [PATCH 11/27] fix: hide sensitive data in some logs --- apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl | 2 +- apps/emqx_coap/src/emqx_coap_resource.erl | 12 ++++++++++-- apps/emqx_exproto/src/emqx_exproto_gsvr.erl | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl index c10efd655..29a0e5d33 100644 --- a/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_adapter.erl @@ -120,7 +120,7 @@ call(Pid, Msg, _) -> init({ClientId, Username, Password, Channel}) -> ?LOG(debug, "try to start adapter ClientId=~p, Username=~p, Password=~p, " - "Channel=~0p", [ClientId, Username, Password, Channel]), + "Channel=~0p", [ClientId, Username, "******", Channel]), State0 = #state{peername = Channel, clientid = ClientId, username = Username, diff --git a/apps/emqx_coap/src/emqx_coap_resource.erl b/apps/emqx_coap/src/emqx_coap_resource.erl index daa536540..2db10dc31 100644 --- a/apps/emqx_coap/src/emqx_coap_resource.erl +++ b/apps/emqx_coap/src/emqx_coap_resource.erl @@ -48,7 +48,7 @@ coap_discover(_Prefix, _Args) -> [{absolute, [<<"mqtt">>], []}]. coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) -> - ?LOG(debug, "coap_get() Path=~p, Query=~p~n", [Path, Query]), + ?LOG(debug, "coap_get() Path=~p, Query=~p~n", [Path, redact_query(Query)]), #coap_mqtt_auth{clientid = Clientid, username = Usr, password = Passwd} = get_auth(Query), case emqx_coap_mqtt_adapter:client_pid(Clientid, Usr, Passwd, ChId) of {ok, Pid} -> @@ -65,7 +65,8 @@ coap_get(ChId, ?MQTT_PREFIX, Path, Query, _Content) -> {error, internal_server_error} end; coap_get(ChId, Prefix, Path, Query, _Content) -> - ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, Path=~p, Query=~p", [ChId, Prefix, Path, Query]), + ?LOG(error, "ignore bad get request ChId=~p, Prefix=~p, Path=~p, Query=~p", + [ChId, Prefix, Path, redact_query(Query)]), {error, bad_request}. coap_post(_ChId, _Prefix, _Topic, _Content) -> @@ -149,3 +150,10 @@ topic([Path | TopicPath]) -> <> end. +redact_query(Auths) -> + lists:map(fun(<<$p, $=, _Rest/binary>>) -> + <<$p, $=, "******">>; + (E) -> + E + end, + Auths). diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl index 91d119c9a..7c6d9d62b 100644 --- a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl @@ -60,7 +60,7 @@ close(Req = #{conn := Conn}, Md) -> authenticate(Req = #{conn := Conn, password := Password, clientinfo := ClientInfo}, Md) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), + ?LOG_SENSITIVE(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), case validate(clientinfo, ClientInfo) of false -> {ok, response({error, ?RESP_REQUIRED_PARAMS_MISSED}), Md}; From 0d2f1eb49e6cef67f82f55833a4d9b682b91cecb Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 22 Nov 2022 17:31:59 +0800 Subject: [PATCH 12/27] chore: update appup --- apps/emqx_exproto/src/emqx_exproto.appup.src | 34 ++++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index e97296a68..2fe600492 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,17 +1,24 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, - {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12", + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11", + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, - [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, @@ -20,17 +27,24 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.12",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, - {"4.3.11",[{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + [{"4.3.12", + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, + {"4.3.11", + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-8]">>, - [{load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, From 3c84f4b2547c8c611b74f39a39a5a58b3bf87652 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Nov 2022 18:14:15 +0100 Subject: [PATCH 13/27] chore: re-generate appup --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++++-- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index e0550fa78..924fffffa 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.3.8"}, % strict semver, bump manually! + {vsn, "4.3.9"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 5f1cd15b0..7fd40959f 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.7", + [{"4.3.8",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[3-6]">>, @@ -9,7 +10,8 @@ {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.7", + [{"4.3.8",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.3.7", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[3-6]">>, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 7989c9542..be6ff4579 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.17"}, % strict semver, bump manually! + {vsn, "4.3.18"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index db60b666c..333c232f8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.16", + [{"4.3.17",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, @@ -314,7 +315,8 @@ {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.16", + [{"4.3.17",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.3.16", [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_cli,brutal_purge,soft_purge,[]}, From 478b006e685cc83807cd71a5cc31fe708880476a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Nov 2022 21:04:53 +0100 Subject: [PATCH 14/27] chore: update versions --- deploy/charts/emqx/Chart.yaml | 4 ++-- include/emqx_release.hrl | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 28cafe9b3..1997c0f27 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.22 +version: 4.3.23 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.3.22 +appVersion: 4.3.23 diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 6c5de4a24..274225ab7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.22"}). +-define(EMQX_RELEASE, {opensource, "4.3.23-alhpa.1"}). -else. diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 07c67545b..cb36b99a8 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.18"}, % strict semver, bump manually! + {vsn, "4.3.19"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, From f3fd90604373c0c5a32d20883ca054e571fdb97f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 26 Nov 2022 21:05:15 +0100 Subject: [PATCH 15/27] chore: bump emqx app vsn --- src/emqx.app.src | 2 +- src/emqx.appup.src | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index 99715d1a2..aa0ff68f4 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.3.23"}, % strict semver, bump manually! + {vsn, "4.3.24"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 6c016f347..377e7353d 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.22", + [{"4.3.23", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + {"4.3.22", [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -877,7 +880,10 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.22", + [{"4.3.23", + [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, + {"4.3.22", [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, From 027c05af99d0b4a6df1cbcde2e51a59e3a3dc712 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Nov 2022 12:19:38 +0100 Subject: [PATCH 16/27] test(emqx_channel): fix two flaky tests --- src/emqx.appup.src | 6 ++++-- src/emqx_channel.erl | 1 - test/emqx_channel_SUITE.erl | 28 ++++++++++++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 377e7353d..e0356aca6 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,7 +2,8 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.23", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.22", [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -881,7 +882,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.23", - [{load_module,emqx_app,brutal_purge,soft_purge,[]}, + [{load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}]}, {"4.3.22", [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 4cae29348..9fad69f90 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -643,7 +643,6 @@ after_message_acked(ClientInfo, Msg, PubAckProps) -> %% Process Subscribe %%-------------------------------------------------------------------- --compile({inline, [process_subscribe/3]}). process_subscribe(TopicFilters, SubProps, Channel) -> process_subscribe(TopicFilters, SubProps, Channel, []). diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl index d46d95ff2..7aa2bef99 100644 --- a/test/emqx_channel_SUITE.erl +++ b/test/emqx_channel_SUITE.erl @@ -321,11 +321,17 @@ t_handle_in_pubcomp_not_found_error(_) -> t_handle_in_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), - Channel = channel(#{conn_state => connected}), - TopicFilters = [{<<"+">>, ?DEFAULT_SUBOPTS}], - Subscribe = ?SUBSCRIBE_PACKET(1, #{}, TopicFilters), - Replies = [{outgoing, ?SUBACK_PACKET(1, [?QOS_0])}, {event, updated}], - {ok, Replies, _Chan} = emqx_channel:handle_in(Subscribe, Channel). + meck:new(emqx_mqtt_caps), + ok = meck:expect(emqx_mqtt_caps, check_sub, fun(_, _, _) -> ok end), + try + Channel = channel(#{conn_state => connected}), + TopicFilters = [{<<"+">>, ?DEFAULT_SUBOPTS}], + Subscribe = ?SUBSCRIBE_PACKET(1, #{}, TopicFilters), + Replies = [{outgoing, ?SUBACK_PACKET(1, [?QOS_0])}, {event, updated}], + {ok, Replies, _Chan} = emqx_channel:handle_in(Subscribe, Channel) + after + meck:unload(emqx_mqtt_caps) + end. t_handle_in_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, @@ -394,9 +400,15 @@ t_process_publish_qos1(_) -> t_process_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), - TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], - {[{TopicFilter, ?RC_SUCCESS}], _Channel} = - emqx_channel:process_subscribe(TopicFilters, #{}, channel()). + meck:new(emqx_mqtt_caps), + ok = meck:expect(emqx_mqtt_caps, check_sub, fun(_, _, _) -> ok end), + try + TopicFilters = [ TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], + {[{TopicFilter, ?RC_SUCCESS}], _Channel} = + emqx_channel:process_subscribe(TopicFilters, #{}, channel()) + after + meck:unload(emqx_mqtt_caps) + end. t_process_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end), From ad024871e12119d30ec64d30769fc31a0cc2cfb8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Nov 2022 13:27:29 +0100 Subject: [PATCH 17/27] ci: fix artifact path for macos --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4d82c4f0e..0ad7e9804 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -145,7 +145,7 @@ jobs: apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v3 with: - name: ${{ env.EMQX_NAME }}-${{ matrix.otp }} + name: ${{ env.EMQX_NAME }} path: _packages/${{ env.EMQX_NAME }}/ linux: From 8b37baad3a6dfb62df7600de988353d064b229cf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Nov 2022 17:33:32 +0100 Subject: [PATCH 18/27] ci: upgrade build-env image to erl23.3.4.18-1 --- .github/workflows/build_slim_packages.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 65f3c760d..e06a74dc4 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: otp: - - erl23.3.4.9-3 + - erl23.3.4.18-1 os: - ubuntu20.04 - centos7 @@ -31,9 +31,7 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - # keep using v1 for now as the otp-23 image has an old version git - # TODO: change to v3 after OTP is upgraded to 23.3.4.18-1 - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: fetch-depth: 0 # clone full git history - name: fix-git-unsafe-repository From 1c1e158874d3f50f52a3e6a8b4904c818259ec72 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 27 Nov 2022 17:42:42 +0100 Subject: [PATCH 19/27] ci: build slim package on centos 8 centos 7 has some git compatibility issues --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index e06a74dc4..6a1bf6e87 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -15,7 +15,7 @@ jobs: - erl23.3.4.18-1 os: - ubuntu20.04 - - centos7 + - centos8 runs-on: - aws-amd64 - ubuntu-20.04 From 2e58152c32e53912bd716d547ab9ed66c810c21e Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 26 Oct 2022 12:57:59 +0200 Subject: [PATCH 20/27] ci: bump otp version to 23.3.4.18-1 --- .github/workflows/apps_version_check.yaml | 2 +- .github/workflows/build_packages.yaml | 12 ++++++------ .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/release.yaml | 2 +- .github/workflows/run_acl_migration_tests.yaml | 2 +- .github/workflows/run_automate_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 10 +++++----- .github/workflows/run_test_cases.yaml | 8 ++++---- 9 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 257eba4f7..b69436901 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: erl_otp: - - erl23.3.4.9-3 + - erl23.3.4.18-1 os: - ubuntu20.04 diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 0ad7e9804..837503294 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -20,7 +20,7 @@ jobs: # avoid building when syncing to ee repo if: endsWith(github.repository, 'emqx') runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 outputs: profiles: ${{ steps.detect-profiles.outputs.profiles}} @@ -58,7 +58,7 @@ jobs: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} otp: - - 23.3.4.13 + - 23.3.4.17 exclude: - profile: emqx-edge @@ -119,7 +119,7 @@ jobs: strategy: matrix: otp: - - 23.3.4.9-3 + - 23.3.4.18-1 os: - macos-11 runs-on: ${{ matrix.os }} @@ -174,7 +174,7 @@ jobs: # - raspbian9 otp_version: #- 23.2.7.2-emqx-3 - - 23.3.4.9-3 + - 23.3.4.18-1 exclude: - os: centos6 arch: arm64 @@ -330,7 +330,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -346,7 +346,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine + BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine RUN_FROM=alpine:3.12 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile.enterprise diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6a1bf6e87..f805b6ee2 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -73,7 +73,7 @@ jobs: strategy: matrix: otp: - - 23.3.4.9-3 + - 23.3.4.18-1 os: - macos-11 runs-on: ${{ matrix.os }} diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 07ec47754..9dedec3dd 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c8065cd0c..205863c98 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 outputs: profiles: ${{ steps.detect-profiles.outputs.profiles}} diff --git a/.github/workflows/run_acl_migration_tests.yaml b/.github/workflows/run_acl_migration_tests.yaml index bbaed018f..9bc801c96 100644 --- a/.github/workflows/run_acl_migration_tests.yaml +++ b/.github/workflows/run_acl_migration_tests.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch jobs: test: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 strategy: fail-fast: true env: diff --git a/.github/workflows/run_automate_tests.yaml b/.github/workflows/run_automate_tests.yaml index 99c243b63..073c884b3 100644 --- a/.github/workflows/run_automate_tests.yaml +++ b/.github/workflows/run_automate_tests.yaml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@v2 - uses: erlef/setup-beam@v1 with: - otp-version: "23.3.4.13" + otp-version: "23.3.4.17" - name: build docker id: build_docker run: | diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index dec677e9c..c16e7435f 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "23.3.4.9" + otp-version: "23.3.4.17" - name: make docker run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -67,7 +67,7 @@ jobs: - uses: actions/checkout@v1 - uses: erlef/setup-beam@v1 with: - otp-version: "23.3.4.9" + otp-version: "23.3.4.17" - name: prepare run: | if make emqx-ee --dry-run > /dev/null 2>&1; then @@ -154,7 +154,7 @@ jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 outputs: profile: ${{ steps.profile-and-versions.outputs.profile }} vsn: ${{ steps.profile-and-versions.outputs.vsn }} @@ -200,7 +200,7 @@ jobs: relup_test_build: needs: relup_test_plan runs-on: ubuntu-20.04 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 defaults: run: shell: bash @@ -235,7 +235,7 @@ jobs: - relup_test_plan - relup_test_build runs-on: ubuntu-20.04 - container: emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 + container: emqx/relup-test-env:erl23.3.4.18-1-ubuntu20.04 strategy: fail-fast: false matrix: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index ea30902bc..9dfca81e4 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -12,7 +12,7 @@ on: jobs: prepare: runs-on: aws-amd64 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 outputs: fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} @@ -58,7 +58,7 @@ jobs: eunit_and_proper: needs: prepare runs-on: aws-amd64 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 strategy: fail-fast: false matrix: @@ -86,7 +86,7 @@ jobs: fast_ct: needs: prepare runs-on: ${{ matrix.runs-on }} - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 strategy: fail-fast: false matrix: @@ -249,7 +249,7 @@ jobs: - fast_ct - docker_ct runs-on: aws-amd64 - container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 + container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04 steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 From cc342e7a096b7e038fd0b5ee340ba8b4fd53eb0c Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 11:00:34 +0100 Subject: [PATCH 21/27] chore: bump version to v4.4.12-alpha.1 --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 12265aa81..7fa90615f 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.4.11"}). +-define(EMQX_RELEASE, {opensource, "4.4.12-alpha.1"}). -else. From 5f10b04971d5110b771af9e63e2231a9f77898c0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 11:02:10 +0100 Subject: [PATCH 22/27] chore: bump chart versions --- deploy/charts/emqx/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 1bd3f411d..a4f2e1a6d 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -13,8 +13,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.4.11 +version: 4.4.12 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 4.4.11 +appVersion: 4.4.12 From 89ad7ed7ef601a90cd9843bc3dea5eb62c612c02 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 11:03:28 +0100 Subject: [PATCH 23/27] chore: bump app versions --- apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src | 2 +- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- lib-ce/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- src/emqx.app.src | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src index 2a50ac509..8b66965b1 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_jwt, [{description, "EMQ X Authentication with JWT"}, - {vsn, "4.4.7"}, % strict semver, bump manually! + {vsn, "4.4.8"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_jwt_sup]}, {applications, [kernel,stdlib,jose]}, diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index b4ea07898..c0648d715 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_coap, [{description, "EMQ X CoAP Gateway"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gen_coap]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 93f2c6fb8..4a5db3094 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.13"}, %% 4.3.3 is used by ee + {vsn, "4.3.14"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index d1860e208..c9c273344 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQX Web Dashboard"}, - {vsn, "4.4.10"}, % strict semver, bump manually! + {vsn, "4.4.11"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index b9c623110..05c9a73ec 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.4.7"}, + {vsn, "4.4.8"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/src/emqx.app.src b/src/emqx.app.src index e9f30de2f..47b9cd084 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -6,7 +6,7 @@ %% the emqx `release' version, which in turn is comprised of several %% apps, one of which is this. See `emqx_release.hrl' for more %% info. - {vsn, "4.4.11"}, % strict semver, bump manually! + {vsn, "4.4.12"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [ kernel From 8998fcc166d706aa00c63230d7f3743f2720bfaa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 11:04:18 +0100 Subject: [PATCH 24/27] chore: update relup paths db --- data/relup-paths.eterm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/relup-paths.eterm b/data/relup-paths.eterm index 3bbfbcb79..a1f7e912c 100644 --- a/data/relup-paths.eterm +++ b/data/relup-paths.eterm @@ -45,6 +45,12 @@ <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>, <<"4.4.10">>], otp => <<"24.3.4.2-1">>}}. +{<<"4.4.12">>, + #{from_versions => + [<<"4.4.0">>,<<"4.4.1">>,<<"4.4.2">>,<<"4.4.3">>,<<"4.4.4">>, + <<"4.4.5">>,<<"4.4.6">>,<<"4.4.7">>,<<"4.4.8">>,<<"4.4.9">>, + <<"4.4.10">>,<<"4.4.11">>], + otp => <<"24.3.4.2-1">>}}. {<<"4.5.0">>, #{from_versions => [<<"4.4.8">>,<<"4.4.9">>,<<"4.4.10">>, <<"4.4.11">>], From d244c298dc5a174768738cd48aff2ac8d4a90739 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 12:49:35 +0100 Subject: [PATCH 25/27] chore: update app vsn and appup --- .../emqx_auth_jwt/src/emqx_auth_jwt.appup.src | 6 ++-- apps/emqx_coap/src/emqx_coap.appup.src | 32 +++++++++++-------- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook.appup.src | 16 +++++++--- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 ++-- .../src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 6 ++-- .../emqx_modules/src/emqx_modules.appup.src | 16 +++++++--- src/emqx.appup.src | 12 +++++-- 9 files changed, 66 insertions(+), 32 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index 80431f300..0c2c0e58f 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.6", + [{"4.4.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.4.6", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {"4.4.5", @@ -18,7 +19,8 @@ {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[0-1]">>,[{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.4.6", + [{"4.4.7",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, + {"4.4.6", [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]}, {"4.4.5", diff --git a/apps/emqx_coap/src/emqx_coap.appup.src b/apps/emqx_coap/src/emqx_coap.appup.src index 1e766c45b..8b840d8a0 100644 --- a/apps/emqx_coap/src/emqx_coap.appup.src +++ b/apps/emqx_coap/src/emqx_coap.appup.src @@ -1,15 +1,19 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{<<"4\\.3\\.[0-1]">>,[ - {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}, - {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []}, - {load_module, emqx_coap_resource, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []}], - [{<<"4\\.3\\.[0-1]">>,[ - {load_module, emqx_coap_mqtt_adapter, brutal_purge, soft_purge, []}, - {load_module, emqx_coap_pubsub_resource, brutal_purge, soft_purge, []}, - {load_module, emqx_coap_resource, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []}] -}. + [{"4.3.2", + [{load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_pubsub_resource,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.2", + [{load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_coap_mqtt_adapter,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_pubsub_resource,brutal_purge,soft_purge,[]}, + {load_module,emqx_coap_resource,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 1e9a01cd5..edf1701be 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.4.4"}, + {vsn, "4.4.5"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index f4ba11f92..6c81649d1 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,9 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.3",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, + [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[1-2]">>, - [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, @@ -15,9 +19,13 @@ {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {update,emqx_exhook_mngr,{advanced,["4.4.0"]}}]}, {<<".*">>,[]}], - [{"4.4.3",[{load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, + [{"4.4.4",[{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}]}, + {"4.4.3", + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}]}, {<<"4\\.4\\.[1-2]">>, - [{load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, + [{load_module,emqx_exhook_pb,brutal_purge,soft_purge,[]}, + {load_module,emqx_exhook_handler,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_server,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook,brutal_purge,soft_purge,[]}, {load_module,emqx_exhook_mngr,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 2fe600492..44585180b 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.12", + [{"4.3.13",[{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}]}, + {"4.3.12", [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.11", @@ -27,7 +28,8 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.12", + [{"4.3.13",[{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}]}, + {"4.3.12", [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}]}, {"4.3.11", diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 31e08f10f..9996a2e46 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.4.11"}, % strict semver, bump manually! + {vsn, "4.4.12"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry, emqx_rule_engine_jwt_sup]}, {applications, [kernel,stdlib,rulesql,getopt,jose]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 1ef25a529..6e62e5df8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.10", + [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.4.10", [{add_module,emqx_rule_engine_jwt}, {add_module,emqx_rule_engine_jwt_worker}, {add_module,emqx_rule_engine_jwt_sup}, @@ -196,7 +197,8 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.10", + [{"4.4.11",[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}]}, + {"4.4.10", [{load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_monitor,brutal_purge,soft_purge,[]}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src index 1cf968166..0abd3a3e9 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.appup.src +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -1,9 +1,13 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.6",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.4.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.6", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.5", - [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}, {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, {"4.4.4", @@ -36,9 +40,13 @@ {load_module,emqx_mod_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.6",[{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, + [{"4.4.7",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]}, + {"4.4.6", + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}]}, {"4.4.5", - [{load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, + [{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}, + {load_module,emqx_mod_delayed,brutal_purge,soft_purge,[]}, {load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}, {load_module,emqx_modules,brutal_purge,soft_purge,[]}]}, {"4.4.4", diff --git a/src/emqx.appup.src b/src/emqx.appup.src index c3d1d5312..278007d81 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,11 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.4.10", + [{"4.4.11", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {"4.4.10", [{add_module,emqx_ocsp_cache}, {add_module,emqx_crl_cache}, {add_module,emqx_const_v1}, @@ -347,7 +351,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.4.10", + [{"4.4.11", + [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + {"4.4.10", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]}, {load_module,emqx_kernel_sup,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, From 1b717ee34999820631cf2bc7b9cb60d90c1a8fa5 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 13:11:50 +0100 Subject: [PATCH 26/27] ci: delete otp version suffix from intermediate artifact name --- .github/workflows/build_packages.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4fe252639..8b19bad44 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -98,7 +98,7 @@ jobs: echo "EQMX uninstaled" - uses: actions/upload-artifact@v3 with: - name: ${{ matrix.profile }}-windows + name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ mac: @@ -209,7 +209,7 @@ jobs: --builder "ghcr.io/emqx/emqx-builder/4.4-20:${OTP}-${SYSTEM}" - uses: actions/upload-artifact@v1 with: - name: ${{ matrix.profile }}-${{ matrix.otp }} + name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ docker: @@ -315,16 +315,11 @@ jobs: fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - otp: - - 24.3.4.2-1 - include: - - profile: emqx - otp: windows # otp version on windows is rather fixed steps: - uses: actions/download-artifact@v2 with: - name: ${{ matrix.profile }}-${{ matrix.otp }} + name: ${{ matrix.profile }} path: packages/${{ matrix.profile }} - name: install dos2unix run: sudo apt-get update && sudo apt install -y dos2unix From 2331c9bd5e54147075a2311e74ae16c9167874f0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 28 Nov 2022 14:32:05 +0100 Subject: [PATCH 27/27] chore: add emqx_cm to appup --- src/emqx.appup.src | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 278007d81..1572fc18e 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -4,6 +4,7 @@ [{"4.4.11", [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.10", [{add_module,emqx_ocsp_cache}, @@ -354,6 +355,7 @@ [{"4.4.11", [{load_module,emqx_relup,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.4.10", [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},