diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 21156957b..73bc9dbb4 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -39,6 +39,8 @@ jobs: fi - name: fix-git-unsafe-repository run: git config --global --add safe.directory /__w/emqx/emqx + - name: make xref + run: make xref - name: build zip packages run: make ${EMQX_NAME}-zip - name: build deb/rpm packages diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index bf9aed8d0..45bcea72e 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -10,6 +10,19 @@ File format: - One list item per change topic Change log ends with a list of github PRs +## v4.3.17 + +### Bug fixes + +- Fixed issue where the dashboard APIs were being exposed under the + management listener. [#8411] + +- Fixed crash when shared persistent subscription [#8441] + +### Enhancements +- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450] +- Add check_conf cli to check config format. [#8486] + ## v4.3.16 ### Enhancements diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 584448046..458a1d0a0 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -139,7 +139,7 @@ auth.http.pool_size = 32 ## Whether to enable HTTP Pipelining ## ## See: https://en.wikipedia.org/wiki/HTTP_pipelining -auth.http.enable_pipelining = true +auth.http.enable_pipelining = 100 ##------------------------------------------------------------------------------ ## SSL options diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index 0f56390e2..627870b58 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -110,10 +110,29 @@ end}. ]}. {mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [ - {default, true}, - {datatype, {enum, [true, false]}} + {default, "100"}, + {datatype, string} ]}. +{translation, "emqx_auth_http.enable_pipelining", fun(Conf) -> + case cuttlefish:conf_get("auth.http.enable_pipelining", Conf, undefined) of + undefined -> 100; + Str -> + try + erlang:list_to_integer(Str) + catch _:_ -> + case erlang:list_to_atom(Str) of + true -> + 100; + false -> + 1; + _ -> + 100 + end + end + end +end}. + {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [ {datatype, string} ]}. diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index fba56740f..e943317f8 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.6"}, % strict semver, bump manually! + {vsn, "4.3.7"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index 519604d24..f5c2bfe42 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,6 +1,10 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.5", + [{"4.3.6", + [ %% There are only changes to the schema file, so we don't need any + %% commands here + ]}, + {"4.3.5", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.4", @@ -19,7 +23,11 @@ {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.5", + [{"4.3.6", + [ %% There are only changes to the schema file, so we don't need any + %% commands here + ]}, + {"4.3.5", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.4", diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 28be88159..95a692bfd 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.4.5"}, % strict semver, bump manually! + {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,emqx_plugin_libs,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index aa3aeb3af..3f420780f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -122,7 +122,7 @@ format({_Subscriber, Topic, Options = #{share := Group}}) -> #{node => node(), topic => filename:join([<<"$share">>, Group, Topic]), clientid => maps:get(subid, Options), qos => QoS}; format({_Subscriber, Topic, Options}) -> QoS = maps:get(qos, Options), - #{node => node(), topic => Topic, clientid => maps:get(subid, Options), qos => QoS}. + #{node => node(), topic => Topic, clientid => maps:get(subid, Options, ""), qos => QoS}. %%-------------------------------------------------------------------- %% Query Function diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 51eb5099e..66ed4484d 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -680,7 +680,7 @@ acl(["cache-clean", "node", Node]) -> with_log(fun() -> for_node(fun emqx_mgmt:clean_acl_cache_all/1, Node) end, "ACL cache drain start"); acl(["cache-clean", "all"]) -> - with_log(fun emqx_mgmt:clean_acl_cache_all/1, + with_log(fun emqx_mgmt:clean_acl_cache_all/0, "ACL cache drain start"); acl(["cache-clean", ClientId]) -> emqx_mgmt:clean_acl_cache(ClientId); diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index bd02146ce..22ac48d3e 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -91,8 +91,8 @@ listener_name(Proto) -> http_handlers() -> Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()), - [{"/api/v4", minirest:handler(#{apps => Plugins ++ - [emqx_plugin_libs, emqx_modules] -- ?EXCEPT_PLUGIN, + [{"/api/v4", minirest:handler(#{apps => (Plugins ++ + [emqx_plugin_libs, emqx_modules]) -- ?EXCEPT_PLUGIN, except => ?EXCEPT, filter => fun ?MODULE:filter/1}), [{authorization, fun ?MODULE:authorize_appid/1}]}]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index f8a82bdf6..cdf2a970b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -9,7 +9,9 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -74,7 +76,10 @@ {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}]}, {"4.4.3", - [{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_rule_validator,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_utils,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 6587f9c5f..223c672e6 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -184,6 +184,17 @@ ]). -export([list_events/2]). +-export([query/3]). + +-define(RULE_QS_SCHEMA, {?RULE_TAB, + [ + {<<"enabled">>, atom}, + {<<"for">>, binary}, + {<<"_like_id">>, binary}, + {<<"_like_for">>, binary}, + {<<"_match_for">>, binary}, + {<<"_like_description">>, binary} + ]}). -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))). -define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))). @@ -261,8 +272,14 @@ update_rule(#{id := Id}, Params) -> return({error, 400, ?ERR_BADARGS(Reason)}) end. -list_rules(_Bindings, _Params) -> - return_all(emqx_rule_registry:get_rules_ordered_by_ts()). +list_rules(_Bindings, Params) -> + case proplists:get_value(<<"enable_paging">>, Params, true) of + true -> + SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end, + return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)}); + false -> + return_all(emqx_rule_registry:get_rules_ordered_by_ts()) + end. show_rule(#{id := Id}, _Params) -> reply_with(fun emqx_rule_registry:get_rule/1, Id). @@ -454,6 +471,7 @@ record_to_map(#rule{id = Id, actions = Actions, on_action_failed = OnFailed, enabled = Enabled, + created_at = CreatedAt, description = Descr}) -> #{id => Id, for => Hook, @@ -462,6 +480,7 @@ record_to_map(#rule{id = Id, on_action_failed => OnFailed, metrics => get_rule_metrics(Id), enabled => Enabled, + created_at => CreatedAt, description => Descr }; @@ -599,3 +618,41 @@ get_action_metrics(Id) -> Res -> [maps:put(node, Node, Res)] end || Node <- ekka_mnesia:running_nodes()]). + +query({Qs, []}, Start, Limit) -> + Ms = qs2ms(Qs), + emqx_mgmt_api:select_table(?RULE_TAB, Ms, Start, Limit, fun record_to_map/1); + +query({Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(?RULE_TAB, MatchFun, Start, Limit, fun record_to_map/1). + +qs2ms(Qs) -> + Init = #rule{for = '_', enabled = '_', _ = '_'}, + MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs), + [{MatchHead, [], ['$_']}]. + +match_ms({for, '=:=', Value}, MatchHead) -> MatchHead#rule{for = Value}; +match_ms({enabled, '=:=', Value}, MatchHead) -> MatchHead#rule{enabled = Value}; +match_ms(_, MatchHead) -> MatchHead. + +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + fun(Rows) -> + Ls = ets:match_spec_run(Rows, MsC), + lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls) + end. + +run_fuzzy_match(_, []) -> true; +run_fuzzy_match(E = #rule{id = Id}, [{id, like, Pattern}|Fuzzy]) -> + binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{description = Desc}, [{description, like, Pattern}|Fuzzy]) -> + binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, match, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = #rule{for = Topics}, [{for, like, Pattern}|Fuzzy]) -> + lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) + andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index ff1b8663e..c62ba06af 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -62,7 +62,8 @@ groups() -> t_show_action_api, t_crud_resources_api, t_list_resource_types_api, - t_show_resource_type_api + t_show_resource_type_api, + t_list_rule_api ]}, {cli, [], [t_rules_cli, @@ -515,6 +516,74 @@ t_crud_rule_api(_Config) -> ?assertMatch({ok, #{code := 404, message := _Message}}, NotFound), ok. +t_list_rule_api(_Config) -> + AddIds = + lists:map(fun(Seq) -> + SeqBin = integer_to_binary(Seq), + {ok, #{code := 0, data := #{id := Id}}} = + emqx_rule_engine_api:create_rule(#{}, + [{<<"name">>, <<"debug-rule-", SeqBin/binary>>}, + {<<"rawsql">>, <<"select * from \"t/a/", SeqBin/binary, "\"">>}, + {<<"actions">>, [[{<<"name">>,<<"inspect">>}, {<<"params">>,[{<<"arg1">>,1}]}]]}, + {<<"description">>, <<"debug rule desc ", SeqBin/binary>>}]), + Id + end, lists:seq(1, 20)), + + {ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}, {<<"enable_paging">>, true}]), + ?assertEqual(10, length(Rules11)), + {ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{}, + [{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}, {<<"enable_paging">>, true}]), + ?assertEqual(10, length(Rules12)), + Rules1 = Rules11 ++ Rules12, + + [RuleID | _] = AddIds, + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"enabled">>, false}]), + Params1 = [{<<"enabled">>,<<"true">>}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)), + + Params2 = [{<<"for">>, RuleID}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2), + ?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)), + + Params3 = [{<<"_like_id">>,<<"rule:">>}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3), + ?assertEqual(length(Rules1), length(Rules4)), + + Params4 = [{<<"_like_for">>,<<"t/a/">>}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assertEqual(length(Rules1), length(Rules5)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t/b/c\"">>}]), + {ok, #{code := 0, data := Rules6}} = emqx_rule_engine_api:list_rules(#{}, Params4), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)), + ?assertEqual(1, length(Rules1) - length(Rules6)), + + Params5 = [{<<"_match_for">>,<<"t/+/+">>}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assertEqual(length(Rules1), length(Rules7)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"rawsql">>, <<"select * from \"t1/b\"">>}]), + {ok, #{code := 0, data := Rules8}} = emqx_rule_engine_api:list_rules(#{}, Params5), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)), + ?assertEqual(1, length(Rules1) - length(Rules8)), + + Params6 = [{<<"_like_description">>,<<"rule">>}, {<<"enable_paging">>, true}], + {ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assertEqual(length(Rules1), length(Rules9)), + {ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID}, + [{<<"description">>, <<"not me">>}]), + {ok, #{code := 0, data := Rules10}} = emqx_rule_engine_api:list_rules(#{}, Params6), + ?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules10)), + ?assertEqual(1, length(Rules1) - length(Rules10)), + + lists:foreach(fun(ID) -> + ?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => ID}, [])) + end, AddIds), + ok. + t_list_actions_api(_Config) -> {ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []), %ct:pal("RList : ~p", [Actions]), diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index fd69f9601..43e28fe1a 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.12"}, % strict semver, bump manually! + {vsn, "4.3.13"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 23136d849..fff4ec08c 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -26,6 +26,8 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[0-2]">>, [{apply,{application,stop,[emqx_web_hook]}}, @@ -52,4 +54,6 @@ {"4.3.11", [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.12", + [{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index ae68e79e1..00094d46d 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -67,10 +67,12 @@ description => #{en => <<"Connection Pool">>, zh => <<"连接池大小"/utf8>>} }, + %% NOTE: In the new version `enable_pipelining` is changed to integer type + %% but it needs to be compatible with the old version, so here keep it as boolean enable_pipelining => #{order => 5, type => boolean, default => true, - title => #{en => <<"Enable Pipelining">>, zh => <<"Enable Pipelining"/utf8>>}, + title => #{en => <<"Enable Pipelining">>, zh => <<"开启 Pipelining"/utf8>>}, description => #{en => <<"Whether to enable HTTP Pipelining">>, zh => <<"是否开启 HTTP Pipelining"/utf8>>} }, diff --git a/bin/emqx b/bin/emqx index 2394484b1..ec3390d58 100755 --- a/bin/emqx +++ b/bin/emqx @@ -13,6 +13,10 @@ RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$RUNNER_ROOT_DIR"/releases/emqx_vars +EMQX_LICENSE_CONF='' +REL_NAME="emqx" +ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin" + RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" @@ -197,6 +201,7 @@ usage() { echo " Up/Down-grade: upgrade | downgrade | install | uninstall" echo " Install info: ertspath | root_dir | versions" echo " Runtime info: pid | ping | versions" + echo " Config check: check_conf" echo " Advanced: console_clean | escript | rpc | rpcterms | eval" echo '' echo "Execute '$REL_NAME COMMAND help' for more information" @@ -339,9 +344,12 @@ trim() { # Function to generate app.config and vm.args generate_config() { - ## Delete the *.siz files first or it cann't start after - ## changing the config 'log.rotation.size' - rm -rf "${RUNNER_LOG_DIR}"/*.siz + check_only="$1" + if [ "$check_only" != "check_only" ]; then + ## Delete the *.siz files first or it cann't start after + ## changing the config 'log.rotation.size' + rm -rf "${RUNNER_LOG_DIR}"/*.siz + fi set +e if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then @@ -393,12 +401,19 @@ generate_config() { fi fi done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then echoerr "Error reading $CONFIG_FILE" exit 1 fi + + if [ "$check_only" = "check_only" ]; then + rm -f "$TMP_ARG_FILE" + rm -f "$CUTTLE_GEN_ARG_FILE" + rm -f "$CONFIG_FILE" + else + mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + fi } # check if a PID is down @@ -489,6 +504,9 @@ case "$1" in foreground) IS_BOOT_COMMAND='yes' ;; + check_conf) + IS_BOOT_COMMAND='yes' + ;; esac @@ -858,6 +876,10 @@ case "$1" in ertspath) echo "$ERTS_PATH" ;; + check_conf) + generate_config "check_only" + echo "$RUNNER_ETC_DIR/emqx.conf is ok" + ;; ctl) assert_node_alive diff --git a/data/emqx_vars b/data/emqx_vars index a872da03a..8ca6bf22d 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -12,12 +12,10 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" +EMQX_DESCRIPTION='{{ emqx_description }}' -EMQX_LICENSE_CONF='' -export EMQX_DESCRIPTION='{{ emqx_description }}' +## Warning: DO NOT create new variables using the above vars in this file, +## as the vars above can be overwritten by the relup scripts later, like: +## REL_VSN="new_version" -## computed vars -REL_NAME="emqx" -ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin" - -## updated vars here +## overwritten vars here diff --git a/etc/emqx.conf b/etc/emqx.conf index 2942362d7..cfacde83c 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1477,7 +1477,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 4e9770dab..29067c712 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,4 +1,5 @@ {application, emqx_dashboard, + [{description, "EMQX Web Dashboard"}, {vsn, "4.4.6"}, % strict semver, bump manually! {modules, []}, diff --git a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl index 73f4be5be..c8bd3a967 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_rewrite.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/logger.hrl"). -ifdef(TEST). -export([ compile/1 @@ -45,6 +46,7 @@ load(RawRules) -> {PubRules, SubRules} = compile(RawRules), + ?LOG(info, "[Rewrite] Load rule pub ~0p sub ~0p", [PubRules, SubRules]), emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}, 1000), emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}, 1000), emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}, 1000). @@ -62,6 +64,7 @@ rewrite_publish(Message = #message{topic = Topic}, Rules) -> {ok, Message#message{topic = match_and_rewrite(Topic, Rules, Binds)}}. unload(_) -> + ?LOG(info, "[Rewrite] Unload"), emqx_hooks:del('client.subscribe', {?MODULE, rewrite_subscribe}), emqx_hooks:del('client.unsubscribe', {?MODULE, rewrite_unsubscribe}), emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}). @@ -93,16 +96,19 @@ match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules], Binds) -> end. rewrite(Topic, MP, Dest, Binds) -> - case re:run(Topic, MP, [{capture, all_but_first, list}]) of - {match, Captured} -> - Vars = lists:zip(["\\$" ++ integer_to_list(I) - || I <- lists:seq(1, length(Captured))], Captured), - iolist_to_binary(lists:foldl( - fun({Var, Val}, Acc) -> - re:replace(Acc, Var, Val, [global]) - end, Dest, Binds ++ Vars)); - nomatch -> Topic - end. + NewTopic = + case re:run(Topic, MP, [{capture, all_but_first, list}]) of + {match, Captured} -> + Vars = lists:zip(["\\$" ++ integer_to_list(I) + || I <- lists:seq(1, length(Captured))], Captured), + iolist_to_binary(lists:foldl( + fun({Var, Val}, Acc) -> + re:replace(Acc, Var, Val, [global]) + end, Dest, Binds ++ Vars)); + nomatch -> Topic + end, + ?LOG(debug, "[Rewrite] topic ~0p, params: ~0p dest topic: ~p", [Topic, Binds, NewTopic]), + NewTopic. fill_client_binds(#{clientid := ClientId, username := Username}) -> filter_client_binds([{"%c", ClientId}, {"%u", Username}]); diff --git a/priv/emqx.schema b/priv/emqx.schema index be333614d..94f124361 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -842,7 +842,7 @@ end}. %% @doc Allow anonymous authentication. {mapping, "allow_anonymous", "emqx.allow_anonymous", [ {default, false}, - {datatype, {enum, [true, false]}} + {datatype, {enum, [true, false, false_quick_deny]}} ]}. %% @doc ACL nomatch. @@ -997,7 +997,7 @@ end}. ]}. {mapping, "zone.$name.allow_anonymous", "emqx.zones", [ - {datatype, {enum, [true, false]}} + {datatype, {enum, [true, false, false_quick_deny]}} ]}. {mapping, "zone.$name.acl_nomatch", "emqx.zones", [ diff --git a/rebar.config b/rebar.config index 7aa72ebc9..4d6dd0d89 100644 --- a/rebar.config +++ b/rebar.config @@ -46,12 +46,12 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} - , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} + , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.4"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}} diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index b014f5531..f4ebfb2a9 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) - EMQX_CE_DASHBOARD_VERSION='v4.3.8' - EMQX_EE_DASHBOARD_VERSION='v4.3.21' + EMQX_CE_DASHBOARD_VERSION='v4.3.9' + EMQX_EE_DASHBOARD_VERSION='v4.3.22' ;; 4.4*) # keep the above 4.3 untouched, otherwise conflicts! diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 3b0c2aee1..ab4fe353d 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -14,15 +14,15 @@ fi case $PROFILE in "emqx") - DIR='broker' + DIR='emqx-ce' EDITION='community' ;; "emqx-ee") - DIR='enterprise' + DIR='emqx-ee' EDITION='enterprise' ;; "emqx-edge") - DIR='edge' + DIR='emqx-edge' EDITION='edge' ;; esac @@ -64,8 +64,9 @@ mkdir -p _upgrade_base pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do - filename="$PROFILE-${tag#[e|v]}-otp$OTP_VSN-$SYSTEM-$ARCH.zip" - url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" + filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" + url="https://packages.emqx.io/$DIR/$tag/$filename" + echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -L -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then echo "downloading base package from ${url} ..." curl -L -o "${filename}" "${url}" diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh index 6b15d6917..3221e0d5c 100755 --- a/scripts/update-appup.sh +++ b/scripts/update-appup.sh @@ -93,6 +93,8 @@ else NEW_COPY='no' fi +TOOL_VERSIONS="$PWD/.tool-versions" + if [ "${SKIP_BUILD_BASE:-no}" = 'yes' ]; then echo "not building relup base ${PREV_DIR_BASE}/${PREV_TAG}" else @@ -104,6 +106,9 @@ else git reset --hard git clean -ffdx git checkout "${PREV_TAG}" + # copy current .tool-versions to ensure same OTP version, even if + # overridden. + cp "$TOOL_VERSIONS" ./ make "$PROFILE" popd fi diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 9fe801743..da289db26 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -210,6 +210,9 @@ {load_module,emqx_relup}]}, {"4.4.1", [{load_module,emqx_broker,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, {load_module,emqx_topic,brutal_purge,soft_purge,[]}, {apply,{emqx_exclusive_subscription,on_delete_module, []}}, @@ -276,6 +279,18 @@ {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, + {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, + {load_module,emqx_frame,brutal_purge,soft_purge,[]}, + {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, + {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_message,brutal_purge,soft_purge,[]}, + {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index 476fe91d4..ae030ebaa 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -33,15 +33,13 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). authenticate(ClientInfo = #{zone := Zone}) -> - AuthResult = default_auth_result(Zone), - case - begin ok = emqx_metrics:inc('client.authenticate'), - emqx_zone:get_env(Zone, bypass_auth_plugins, false) - end - of - true -> + ok = emqx_metrics:inc('client.authenticate'), + Username = maps:get(username, ClientInfo, undefined), + {MaybeStop, AuthResult} = default_auth_result(Username, Zone), + case MaybeStop of + stop -> return_auth_result(AuthResult); - false -> + continue -> return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult)) end. @@ -91,10 +89,29 @@ inc_acl_metrics(cache_hit) -> emqx_metrics:inc('client.acl.cache_hit'). %% Auth -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} +default_auth_result(Username, Zone) -> + IsAnonymous = (Username =:= undefined orelse Username =:= <<>>), + AllowAnonymous = emqx_zone:get_env(Zone, allow_anonymous, false), + Bypass = emqx_zone:get_env(Zone, bypass_auth_plugins, false), + %% the `anonymous` field in auth result does not mean the client is + %% connected without username, but if the auth result is based on + %% allowing anonymous access. + IsResultBasedOnAllowAnonymous = + case AllowAnonymous of + true -> true; + _ -> false + end, + Result = case AllowAnonymous of + true -> #{auth_result => success, anonymous => IsResultBasedOnAllowAnonymous}; + _ -> #{auth_result => not_authorized, anonymous => IsResultBasedOnAllowAnonymous} + end, + case {IsAnonymous, AllowAnonymous} of + {true, false_quick_deny} -> + {stop, Result}; + _ when Bypass -> + {stop, Result}; + _ -> + {continue, Result} end. -compile({inline, [return_auth_result/1]}). diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl index c7fb1a3cb..1e77b6014 100644 --- a/src/emqx_shared_sub.erl +++ b/src/emqx_shared_sub.erl @@ -242,7 +242,7 @@ maybe_nack_dropped(Msg) -> %% For fresh Ref we send a nack and return true, to note that the inflight is full {Sender, {fresh, _Group, Ref}} -> nack(Sender, Ref, dropped), drop; - + %% For retry Ref we can't reject a message if inflight is full, so we mark it as %% acknowledged and put it into mqueue {_Sender, {retry, _Group, _Ref}} -> maybe_ack(Msg), store; @@ -311,7 +311,7 @@ do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) -> false; [] -> %% We try redispatch to subs who dropped the message because inflight was full. - Found = maps_find_by(FailedSubs, fun({SubPid, FailReason}) -> + Found = maps_find_by(FailedSubs, fun(SubPid, FailReason) -> FailReason == dropped andalso is_alive_sub(SubPid) end), case Found of diff --git a/test/emqx_access_control_SUITE.erl b/test/emqx_access_control_SUITE.erl index 8b0d5778b..a03312853 100644 --- a/test/emqx_access_control_SUITE.erl +++ b/test/emqx_access_control_SUITE.erl @@ -38,6 +38,12 @@ t_authenticate(_) -> emqx_zone:set_env(zone, allow_anonymous, true), ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). +t_authenticate_fast_fail(_) -> + emqx_zone:set_env(zone, allow_anonymous, false_quick_deny), + ?assertMatch({error, _}, emqx_access_control:authenticate(clientinfo())), + emqx_zone:set_env(zone, allow_anonymous, true), + ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). + t_check_acl(_) -> emqx_zone:set_env(zone, acl_nomatch, deny), application:set_env(emqx, enable_acl_cache, false),