yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8503 from zhongwencool/sync-4.3-to-4.4 

chore: sync 4.3 to 4.4
This commit is contained in:
zhongwencool 2022-07-19 19:38:38 +08:00 committed by GitHub
parent f3ff73e683 347f071079
commit 1a980dfec7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 331 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build_slim_packages.yaml vendored
View File

@ -39,6 +39,8 @@ jobs:
fi
- name: fix-git-unsafe-repository
run: git config --global --add safe.directory /__w/emqx/emqx
- name: make xref
run: make xref
- name: build zip packages
run: make ${EMQX_NAME}-zip
- name: build deb/rpm packages

13
CHANGES-4.3.md
View File

@ -10,6 +10,19 @@ File format:
- One list item per change topic
Change log ends with a list of github PRs
## v4.3.17
### Bug fixes
- Fixed issue where the dashboard APIs were being exposed under the
management listener. [#8411]
- Fixed crash when shared persistent subscription [#8441]
### Enhancements
- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
- Add check_conf cli to check config format. [#8486]
## v4.3.16
### Enhancements

2
apps/emqx_auth_http/etc/emqx_auth_http.conf
View File

@ -139,7 +139,7 @@ auth.http.pool_size = 32
## Whether to enable HTTP Pipelining
##
## See: https://en.wikipedia.org/wiki/HTTP_pipelining
auth.http.enable_pipelining = true
auth.http.enable_pipelining = 100
##------------------------------------------------------------------------------
## SSL options

23
apps/emqx_auth_http/priv/emqx_auth_http.schema
View File

@ -110,10 +110,29 @@ end}.
]}.
{mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [
{default, true},
{datatype, {enum, [true, false]}}
{default, "100"},
{datatype, string}
]}.
{translation, "emqx_auth_http.enable_pipelining", fun(Conf) ->
case cuttlefish:conf_get("auth.http.enable_pipelining", Conf, undefined) of
undefined -> 100;
Str ->
try
erlang:list_to_integer(Str)
catch _:_ ->
case erlang:list_to_atom(Str) of
true ->
100;
false ->
1;
_ ->
100
end
end
end
end}.
{mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [
{datatype, string}
]}.

2
apps/emqx_auth_http/src/emqx_auth_http.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_auth_http,
[{description, "EMQ X Authentication/ACL with HTTP API"},
{vsn, "4.3.6"}, % strict semver, bump manually!
{vsn, "4.3.7"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_auth_http_sup]},
{applications, [kernel,stdlib,ehttpc]},

12
apps/emqx_auth_http/src/emqx_auth_http.appup.src
View File

@ -1,6 +1,10 @@
%% -*- mode: erlang -*-
{VSN,
[{"4.3.5",
[{"4.3.6",
[ %% There are only changes to the schema file, so we don't need any
%% commands here
]},
{"4.3.5",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.4",
@ -19,7 +23,11 @@
{<<"4.3.[0-1]">>,
[{restart_application,emqx_auth_http}]},
{<<".*">>,[]}],
[{"4.3.5",
[{"4.3.6",
[ %% There are only changes to the schema file, so we don't need any
%% commands here
]},
{"4.3.5",
[{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
{load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
{"4.3.4",

2
apps/emqx_management/src/emqx_management.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_management,
[{description, "EMQ X Management API and CLI"},
{vsn, "4.4.5"}, % strict semver, bump manually!
{vsn, "4.4.6"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel,stdlib,emqx_plugin_libs,minirest]},

2
apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl
View File

@ -122,7 +122,7 @@ format({_Subscriber, Topic, Options = #{share := Group}}) ->
#{node => node(), topic => filename:join([<<"$share">>, Group, Topic]), clientid => maps:get(subid, Options), qos => QoS};
format({_Subscriber, Topic, Options}) ->
QoS = maps:get(qos, Options),
#{node => node(), topic => Topic, clientid => maps:get(subid, Options), qos => QoS}.
#{node => node(), topic => Topic, clientid => maps:get(subid, Options, ""), qos => QoS}.
%%--------------------------------------------------------------------
%% Query Function

2
apps/emqx_management/src/emqx_mgmt_cli.erl
View File

@ -680,7 +680,7 @@ acl(["cache-clean", "node", Node]) ->
with_log(fun() -> for_node(fun emqx_mgmt:clean_acl_cache_all/1, Node) end,
"ACL cache drain start");
acl(["cache-clean", "all"]) ->
with_log(fun emqx_mgmt:clean_acl_cache_all/1,
with_log(fun emqx_mgmt:clean_acl_cache_all/0,
"ACL cache drain start");
acl(["cache-clean", ClientId]) ->
emqx_mgmt:clean_acl_cache(ClientId);

4
apps/emqx_management/src/emqx_mgmt_http.erl
View File

@ -91,8 +91,8 @@ listener_name(Proto) ->
http_handlers() ->
Plugins = lists:map(fun(Plugin) -> Plugin#plugin.name end, emqx_plugins:list()),
[{"/api/v4", minirest:handler(#{apps => Plugins ++
[emqx_plugin_libs, emqx_modules] -- ?EXCEPT_PLUGIN,
[{"/api/v4", minirest:handler(#{apps => (Plugins ++
[emqx_plugin_libs, emqx_modules]) -- ?EXCEPT_PLUGIN,
except => ?EXCEPT,
filter => fun ?MODULE:filter/1}),
[{authorization, fun ?MODULE:authorize_appid/1}]}].

2
apps/emqx_rule_engine/src/emqx_rule_engine.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_rule_engine,
[{description, "EMQ X Rule Engine"},
{vsn, "4.4.5"}, % strict semver, bump manually!
{vsn, "4.4.6"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_registry]},
{applications, [kernel,stdlib,rulesql,getopt]},

16
apps/emqx_rule_engine/src/emqx_rule_engine.appup.src
View File

@ -1,8 +1,12 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.4",
[{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
[{"4.4.5",
[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{"4.4.4",
[{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},
@ -66,8 +70,12 @@
{load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.4",
[{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
[{"4.4.5",
[{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]},
{"4.4.4",
[{load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_validator,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_utils,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]},
{load_module,emqx_rule_engine,brutal_purge,soft_purge,[]},

61
apps/emqx_rule_engine/src/emqx_rule_engine_api.erl
View File

@ -184,6 +184,17 @@
]).
-export([list_events/2]).
-export([query/3]).
-define(RULE_QS_SCHEMA, {?RULE_TAB,
[
{<<"enabled">>, atom},
{<<"for">>, binary},
{<<"_like_id">>, binary},
{<<"_like_for">>, binary},
{<<"_match_for">>, binary},
{<<"_like_description">>, binary}
]}).
-define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~s Not Found", [(ID)]))).
-define(ERR_NO_ACTION(NAME), list_to_binary(io_lib:format("Action ~s Not Found", [(NAME)]))).
@ -261,8 +272,14 @@ update_rule(#{id := Id}, Params) ->
return({error, 400, ?ERR_BADARGS(Reason)})
end.
list_rules(_Bindings, _Params) ->
return_all(emqx_rule_registry:get_rules_ordered_by_ts()).
list_rules(_Bindings, Params) ->
case proplists:get_value(<<"enable_paging">>, Params, true) of
true ->
SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
return({ok, emqx_mgmt_api:node_query(node(), Params, ?RULE_QS_SCHEMA, {?MODULE, query}, SortFun)});
false ->
return_all(emqx_rule_registry:get_rules_ordered_by_ts())
end.
show_rule(#{id := Id}, _Params) ->
reply_with(fun emqx_rule_registry:get_rule/1, Id).
@ -454,6 +471,7 @@ record_to_map(#rule{id = Id,
actions = Actions,
on_action_failed = OnFailed,
enabled = Enabled,
created_at = CreatedAt,
description = Descr}) ->
#{id => Id,
for => Hook,
@ -462,6 +480,7 @@ record_to_map(#rule{id = Id,
on_action_failed => OnFailed,
metrics => get_rule_metrics(Id),
enabled => Enabled,
created_at => CreatedAt,
description => Descr
};
@ -599,3 +618,41 @@ get_action_metrics(Id) ->
Res -> [maps:put(node, Node, Res)]
end
|| Node <- ekka_mnesia:running_nodes()]).
query({Qs, []}, Start, Limit) ->
Ms = qs2ms(Qs),
emqx_mgmt_api:select_table(?RULE_TAB, Ms, Start, Limit, fun record_to_map/1);
query({Qs, Fuzzy}, Start, Limit) ->
Ms = qs2ms(Qs),
MatchFun = match_fun(Ms, Fuzzy),
emqx_mgmt_api:traverse_table(?RULE_TAB, MatchFun, Start, Limit, fun record_to_map/1).
qs2ms(Qs) ->
Init = #rule{for = '_', enabled = '_', _ = '_'},
MatchHead = lists:foldl(fun(Q, Acc) -> match_ms(Q, Acc) end, Init, Qs),
[{MatchHead, [], ['$_']}].
match_ms({for, '=:=', Value}, MatchHead) -> MatchHead#rule{for = Value};
match_ms({enabled, '=:=', Value}, MatchHead) -> MatchHead#rule{enabled = Value};
match_ms(_, MatchHead) -> MatchHead.
match_fun(Ms, Fuzzy) ->
MsC = ets:match_spec_compile(Ms),
fun(Rows) ->
Ls = ets:match_spec_run(Rows, MsC),
lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls)
end.
run_fuzzy_match(_, []) -> true;
run_fuzzy_match(E = #rule{id = Id}, [{id, like, Pattern}|Fuzzy]) ->
binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = #rule{description = Desc}, [{description, like, Pattern}|Fuzzy]) ->
binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = #rule{for = Topics}, [{for, match, Pattern}|Fuzzy]) ->
lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics)
andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(E = #rule{for = Topics}, [{for, like, Pattern}|Fuzzy]) ->
lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics)
andalso run_fuzzy_match(E, Fuzzy);
run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false.

71
apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl
View File

@ -62,7 +62,8 @@ groups() ->
t_show_action_api,
t_crud_resources_api,
t_list_resource_types_api,
t_show_resource_type_api
t_show_resource_type_api,
t_list_rule_api
]},
{cli, [],
[t_rules_cli,
@ -515,6 +516,74 @@ t_crud_rule_api(_Config) ->
?assertMatch({ok, #{code := 404, message := _Message}}, NotFound),
ok.
t_list_rule_api(_Config) ->
AddIds =
lists:map(fun(Seq) ->
SeqBin = integer_to_binary(Seq),
{ok, #{code := 0, data := #{id := Id}}} =
emqx_rule_engine_api:create_rule(#{},
[{<<"name">>, <<"debug-rule-", SeqBin/binary>>},
{<<"rawsql">>, <<"select * from \"t/a/", SeqBin/binary, "\"">>},
{<<"actions">>, [[{<<"name">>,<<"inspect">>}, {<<"params">>,[{<<"arg1">>,1}]}]]},
{<<"description">>, <<"debug rule desc ", SeqBin/binary>>}]),
Id
end, lists:seq(1, 20)),
{ok, #{code := 0, data := Rules11}} = emqx_rule_engine_api:list_rules(#{},
[{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"1">>}, {<<"enable_paging">>, true}]),
?assertEqual(10, length(Rules11)),
{ok, #{code := 0, data := Rules12}} = emqx_rule_engine_api:list_rules(#{},
[{<<"_limit">>,<<"10">>}, {<<"_page">>, <<"2">>}, {<<"enable_paging">>, true}]),
?assertEqual(10, length(Rules12)),
Rules1 = Rules11 ++ Rules12,
[RuleID | _] = AddIds,
{ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
[{<<"enabled">>, false}]),
Params1 = [{<<"enabled">>,<<"true">>}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules2}} = emqx_rule_engine_api:list_rules(#{}, Params1),
?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules2)),
Params2 = [{<<"for">>, RuleID}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules3}} = emqx_rule_engine_api:list_rules(#{}, Params2),
?assert(lists:all(fun(#{id := ID}) -> ID =:= RuleID end, Rules3)),
Params3 = [{<<"_like_id">>,<<"rule:">>}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules4}} = emqx_rule_engine_api:list_rules(#{}, Params3),
?assertEqual(length(Rules1), length(Rules4)),
Params4 = [{<<"_like_for">>,<<"t/a/">>}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules5}} = emqx_rule_engine_api:list_rules(#{}, Params4),
?assertEqual(length(Rules1), length(Rules5)),
{ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
[{<<"rawsql">>, <<"select * from \"t/b/c\"">>}]),
{ok, #{code := 0, data := Rules6}} = emqx_rule_engine_api:list_rules(#{}, Params4),
?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules6)),
?assertEqual(1, length(Rules1) - length(Rules6)),
Params5 = [{<<"_match_for">>,<<"t/+/+">>}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules7}} = emqx_rule_engine_api:list_rules(#{}, Params5),
?assertEqual(length(Rules1), length(Rules7)),
{ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
[{<<"rawsql">>, <<"select * from \"t1/b\"">>}]),
{ok, #{code := 0, data := Rules8}} = emqx_rule_engine_api:list_rules(#{}, Params5),
?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules8)),
?assertEqual(1, length(Rules1) - length(Rules8)),
Params6 = [{<<"_like_description">>,<<"rule">>}, {<<"enable_paging">>, true}],
{ok, #{code := 0, data := Rules9}} = emqx_rule_engine_api:list_rules(#{}, Params6),
?assertEqual(length(Rules1), length(Rules9)),
{ok, #{code := 0}} = emqx_rule_engine_api:update_rule(#{id => RuleID},
[{<<"description">>, <<"not me">>}]),
{ok, #{code := 0, data := Rules10}} = emqx_rule_engine_api:list_rules(#{}, Params6),
?assert(lists:all(fun(#{id := ID}) -> ID =/= RuleID end, Rules10)),
?assertEqual(1, length(Rules1) - length(Rules10)),
lists:foreach(fun(ID) ->
?assertMatch({ok, #{code := 0}}, emqx_rule_engine_api:delete_rule(#{id => ID}, []))
end, AddIds),
ok.
t_list_actions_api(_Config) ->
{ok, #{code := 0, data := Actions}} = emqx_rule_engine_api:list_actions(#{}, []),
%ct:pal("RList : ~p", [Actions]),

2
apps/emqx_web_hook/src/emqx_web_hook.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_web_hook,
[{description, "EMQ X WebHook Plugin"},
{vsn, "4.3.12"}, % strict semver, bump manually!
{vsn, "4.3.13"}, % strict semver, bump manually!
{modules, []},
{registered, [emqx_web_hook_sup]},
{applications, [kernel,stdlib,ehttpc]},

4
apps/emqx_web_hook/src/emqx_web_hook.appup.src
View File

@ -26,6 +26,8 @@
{"4.3.11",
[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]},
{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
{"4.3.12",
[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{<<"4\\.3\\.[0-2]">>,
[{apply,{application,stop,[emqx_web_hook]}},
@ -52,4 +54,6 @@
{"4.3.11",
[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]},
{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]},
{"4.3.12",
[{load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}]}.

4
apps/emqx_web_hook/src/emqx_web_hook_actions.erl
View File

@ -67,10 +67,12 @@
description => #{en => <<"Connection Pool">>,
zh => <<"连接池大小"/utf8>>}
},
%% NOTE: In the new version `enable_pipelining` is changed to integer type
%% but it needs to be compatible with the old version, so here keep it as boolean
enable_pipelining => #{order => 5,
type => boolean,
default => true,
title => #{en => <<"Enable Pipelining">>, zh => <<"Enable Pipelining"/utf8>>},
title => #{en => <<"Enable Pipelining">>, zh => <<"开启 Pipelining"/utf8>>},
description => #{en => <<"Whether to enable HTTP Pipelining">>,
zh => <<"是否开启 HTTP Pipelining"/utf8>>}
},

30
bin/emqx
View File

@ -13,6 +13,10 @@ RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090
. "$RUNNER_ROOT_DIR"/releases/emqx_vars
EMQX_LICENSE_CONF=''
REL_NAME="emqx"
ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN"
@ -197,6 +201,7 @@ usage() {
echo " Up/Down-grade: upgrade | downgrade | install | uninstall"
echo " Install info: ertspath | root_dir | versions"
echo " Runtime info: pid | ping | versions"
echo " Config check: check_conf"
echo " Advanced: console_clean | escript | rpc | rpcterms | eval"
echo ''
echo "Execute '$REL_NAME COMMAND help' for more information"
@ -339,9 +344,12 @@ trim() {
# Function to generate app.config and vm.args
generate_config() {
## Delete the *.siz files first or it cann't start after
## changing the config 'log.rotation.size'
rm -rf "${RUNNER_LOG_DIR}"/*.siz
check_only="$1"
if [ "$check_only" != "check_only" ]; then
## Delete the *.siz files first or it cann't start after
## changing the config 'log.rotation.size'
rm -rf "${RUNNER_LOG_DIR}"/*.siz
fi
set +e
if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then
@ -393,12 +401,19 @@ generate_config() {
fi
fi
done
mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
if ! relx_nodetool chkconfig -config "$CONFIG_FILE"; then
echoerr "Error reading $CONFIG_FILE"
exit 1
fi
if [ "$check_only" = "check_only" ]; then
rm -f "$TMP_ARG_FILE"
rm -f "$CUTTLE_GEN_ARG_FILE"
rm -f "$CONFIG_FILE"
else
mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
fi
}
# check if a PID is down
@ -489,6 +504,9 @@ case "$1" in
foreground)
IS_BOOT_COMMAND='yes'
;;
check_conf)
IS_BOOT_COMMAND='yes'
;;
esac
@ -858,6 +876,10 @@ case "$1" in
ertspath)
echo "$ERTS_PATH"
;;
check_conf)
generate_config "check_only"
echo "$RUNNER_ETC_DIR/emqx.conf is ok"
;;
ctl)
assert_node_alive

12
data/emqx_vars
View File

@ -12,12 +12,10 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}"
RUNNER_ETC_DIR="{{ runner_etc_dir }}"
RUNNER_DATA_DIR="{{ runner_data_dir }}"
RUNNER_USER="{{ runner_user }}"
EMQX_DESCRIPTION='{{ emqx_description }}'
EMQX_LICENSE_CONF=''
export EMQX_DESCRIPTION='{{ emqx_description }}'
## Warning: DO NOT create new variables using the above vars in this file,
## as the vars above can be overwritten by the relup scripts later, like:
## REL_VSN="new_version"
## computed vars
REL_NAME="emqx"
ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
## updated vars here
## overwritten vars here

2
etc/emqx.conf
View File

@ -1477,7 +1477,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem
## are used during server authentication and when building the client certificate chain.
##
## Value: File
## listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem
listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem
## The Ephemeral Diffie-Helman key exchange is a very effective way of
## ensuring Forward Secrecy by exchanging a set of keys that never hit

1
lib-ce/emqx_dashboard/src/emqx_dashboard.app.src
View File

@ -1,4 +1,5 @@
{application, emqx_dashboard,
[{description, "EMQX Web Dashboard"},
{vsn, "4.4.6"}, % strict semver, bump manually!
{modules, []},

26
lib-ce/emqx_modules/src/emqx_mod_rewrite.erl
View File

@ -20,6 +20,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx/include/logger.hrl").
-ifdef(TEST).
-export([ compile/1
@ -45,6 +46,7 @@
load(RawRules) ->
{PubRules, SubRules} = compile(RawRules),
?LOG(info, "[Rewrite] Load rule pub ~0p sub ~0p", [PubRules, SubRules]),
emqx_hooks:put('client.subscribe', {?MODULE, rewrite_subscribe, [SubRules]}, 1000),
emqx_hooks:put('client.unsubscribe', {?MODULE, rewrite_unsubscribe, [SubRules]}, 1000),
emqx_hooks:put('message.publish', {?MODULE, rewrite_publish, [PubRules]}, 1000).
@ -62,6 +64,7 @@ rewrite_publish(Message = #message{topic = Topic}, Rules) ->
{ok, Message#message{topic = match_and_rewrite(Topic, Rules, Binds)}}.
unload(_) ->
?LOG(info, "[Rewrite] Unload"),
emqx_hooks:del('client.subscribe', {?MODULE, rewrite_subscribe}),
emqx_hooks:del('client.unsubscribe', {?MODULE, rewrite_unsubscribe}),
emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}).
@ -93,16 +96,19 @@ match_and_rewrite(Topic, [{rewrite, Filter, MP, Dest} | Rules], Binds) ->
end.
rewrite(Topic, MP, Dest, Binds) ->
case re:run(Topic, MP, [{capture, all_but_first, list}]) of
{match, Captured} ->
Vars = lists:zip(["\\$" ++ integer_to_list(I)
|| I <- lists:seq(1, length(Captured))], Captured),
iolist_to_binary(lists:foldl(
fun({Var, Val}, Acc) ->
re:replace(Acc, Var, Val, [global])
end, Dest, Binds ++ Vars));
nomatch -> Topic
end.
NewTopic =
case re:run(Topic, MP, [{capture, all_but_first, list}]) of
{match, Captured} ->
Vars = lists:zip(["\\$" ++ integer_to_list(I)
|| I <- lists:seq(1, length(Captured))], Captured),
iolist_to_binary(lists:foldl(
fun({Var, Val}, Acc) ->
re:replace(Acc, Var, Val, [global])
end, Dest, Binds ++ Vars));
nomatch -> Topic
end,
?LOG(debug, "[Rewrite] topic ~0p, params: ~0p dest topic: ~p", [Topic, Binds, NewTopic]),
NewTopic.
fill_client_binds(#{clientid := ClientId, username := Username}) ->
filter_client_binds([{"%c", ClientId}, {"%u", Username}]);

2
lib-ce/emqx_modules/src/emqx_modules.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_modules,
[{description, "EMQ X Module Management"},
{vsn, "4.4.4"},
{vsn, "4.4.5"},
{modules, []},
{applications, [kernel,stdlib]},
{mod, {emqx_modules_app, []}},

6
lib-ce/emqx_modules/src/emqx_modules.appup.src
View File

@ -1,7 +1,8 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
[{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
{"4.4.2",
[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},
@ -19,7 +20,8 @@
{load_module,emqx_mod_sup,brutal_purge,soft_purge,[]},
{load_module,emqx_mod_trace_api,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
[{"4.4.4",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
{"4.4.3",[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]}]},
{"4.4.2",
[{load_module,emqx_mod_rewrite,brutal_purge,soft_purge,[]},
{load_module,emqx_modules,brutal_purge,soft_purge,[]}]},

4
priv/emqx.schema
View File

@ -842,7 +842,7 @@ end}.
%% @doc Allow anonymous authentication.
{mapping, "allow_anonymous", "emqx.allow_anonymous", [
{default, false},
{datatype, {enum, [true, false]}}
{datatype, {enum, [true, false, false_quick_deny]}}
]}.
%% @doc ACL nomatch.
@ -997,7 +997,7 @@ end}.
]}.
{mapping, "zone.$name.allow_anonymous", "emqx.zones", [
{datatype, {enum, [true, false]}}
{datatype, {enum, [true, false, false_quick_deny]}}
]}.
{mapping, "zone.$name.acl_nomatch", "emqx.zones", [

4
rebar.config
View File

@ -46,12 +46,12 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.6"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.10"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.11"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.7.1"}}}
, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
, {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}}
, {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.4"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}}
, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}}
, {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.5"}}}

4
scripts/get-dashboard.sh
View File

@ -8,8 +8,8 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
case "${PKG_VSN}" in
4.3*)
EMQX_CE_DASHBOARD_VERSION='v4.3.8'
EMQX_EE_DASHBOARD_VERSION='v4.3.21'
EMQX_CE_DASHBOARD_VERSION='v4.3.9'
EMQX_EE_DASHBOARD_VERSION='v4.3.22'
;;
4.4*)
# keep the above 4.3 untouched, otherwise conflicts!

5
scripts/update-appup.sh
View File

@ -93,6 +93,8 @@ else
NEW_COPY='no'
fi
TOOL_VERSIONS="$PWD/.tool-versions"
if [ "${SKIP_BUILD_BASE:-no}" = 'yes' ]; then
echo "not building relup base ${PREV_DIR_BASE}/${PREV_TAG}"
else
@ -104,6 +106,9 @@ else
git reset --hard
git clean -ffdx
git checkout "${PREV_TAG}"
# copy current .tool-versions to ensure same OTP version, even if
# overridden.
cp "$TOOL_VERSIONS" ./
make "$PROFILE"
popd
fi

2
src/emqx.app.src
View File

@ -6,7 +6,7 @@
%% the emqx `release' version, which in turn is comprised of several
%% apps, one of which is this. See `emqx_release.hrl' for more
%% info.
{vsn, "4.4.5"}, % strict semver, bump manually!
{vsn, "4.4.6"}, % strict semver, bump manually!
{modules, []},
{registered, []},
{applications, [ kernel

34
src/emqx.appup.src
View File

@ -1,8 +1,16 @@
%% -*- mode: erlang -*-
%% Unless you know what you are doing, DO NOT edit manually!!
{VSN,
[{"4.4.4",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{"4.4.5",
[
{load_module,emqx_relup,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]},
{"4.4.4",
[{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_broker,brutal_purge,soft_purge,[]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_ctl,brutal_purge,soft_purge,[]},
@ -144,14 +152,22 @@
{load_module,emqx_message,brutal_purge,soft_purge,[]},
{load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
{<<".*">>,[]}],
[{"4.4.4",
[{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
[{"4.4.5",
[
{load_module,emqx_relup,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]},
{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}]},
{"4.4.4",
[{load_module,emqx_access_control,brutal_purge,soft_purge,[]},
{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
{load_module,emqx_access_rule,brutal_purge,soft_purge,[]},
{load_module,emqx_broker,brutal_purge,soft_purge,[]},
{load_module,emqx_channel,brutal_purge,soft_purge,[]},
{load_module,emqx_ctl,brutal_purge,soft_purge,[]},
{load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
{load_module,emqx_topic,brutal_purge,soft_purge,[]},
{apply,{emqx_exclusive_subscription,on_delete_module, []}},
{apply,{emqx_exclusive_subscription,on_delete_module,[]}},
{delete_module,emqx_exclusive_subscription},
{load_module,emqx_relup,brutal_purge,soft_purge,[]},
{load_module,emqx_app,brutal_purge,soft_purge,[]},
@ -162,7 +178,7 @@
{load_module,emqx_ctl,brutal_purge,soft_purge,[]},
{load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
{load_module,emqx_topic,brutal_purge,soft_purge,[]},
{apply,{emqx_exclusive_subscription,on_delete_module, []}},
{apply,{emqx_exclusive_subscription,on_delete_module,[]}},
{delete_module,emqx_exclusive_subscription},
{load_module,emqx_plugins,brutal_purge,soft_purge,[]},
{load_module,emqx_packet,brutal_purge,soft_purge,[]},
@ -186,7 +202,7 @@
{load_module,emqx_ctl,brutal_purge,soft_purge,[]},
{load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
{load_module,emqx_topic,brutal_purge,soft_purge,[]},
{apply,{emqx_exclusive_subscription,on_delete_module, []}},
{apply,{emqx_exclusive_subscription,on_delete_module,[]}},
{delete_module,emqx_exclusive_subscription},
{load_module,emqx_packet,brutal_purge,soft_purge,[]},
{delete_module,emqx_calendar},
@ -212,7 +228,7 @@
[{load_module,emqx_broker,brutal_purge,soft_purge,[]},
{load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
{load_module,emqx_topic,brutal_purge,soft_purge,[]},
{apply,{emqx_exclusive_subscription,on_delete_module, []}},
{apply,{emqx_exclusive_subscription,on_delete_module,[]}},
{delete_module,emqx_exclusive_subscription},
{delete_module,emqx_calendar},
{load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
@ -247,7 +263,7 @@
[{load_module,emqx_broker,brutal_purge,soft_purge,[]},
{load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
{load_module,emqx_topic,brutal_purge,soft_purge,[]},
{apply,{emqx_exclusive_subscription,on_delete_module, []}},
{apply,{emqx_exclusive_subscription,on_delete_module,[]}},
{delete_module,emqx_exclusive_subscription},
{load_module,emqx_packet,brutal_purge,soft_purge,[]},
{delete_module,emqx_calendar},

41
src/emqx_access_control.erl
View File

@ -33,15 +33,13 @@
-spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}).
authenticate(ClientInfo = #{zone := Zone}) ->
AuthResult = default_auth_result(Zone),
case
begin ok = emqx_metrics:inc('client.authenticate'),
emqx_zone:get_env(Zone, bypass_auth_plugins, false)
end
of
true ->
ok = emqx_metrics:inc('client.authenticate'),
Username = maps:get(username, ClientInfo, undefined),
{MaybeStop, AuthResult} = default_auth_result(Username, Zone),
case MaybeStop of
stop ->
return_auth_result(AuthResult);
false ->
continue ->
return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult))
end.
@ -91,10 +89,29 @@ inc_acl_metrics(cache_hit) ->
emqx_metrics:inc('client.acl.cache_hit').
%% Auth
default_auth_result(Zone) ->
case emqx_zone:get_env(Zone, allow_anonymous, false) of
true -> #{auth_result => success, anonymous => true};
false -> #{auth_result => not_authorized, anonymous => false}
default_auth_result(Username, Zone) ->
IsAnonymous = (Username =:= undefined orelse Username =:= <<>>),
AllowAnonymous = emqx_zone:get_env(Zone, allow_anonymous, false),
Bypass = emqx_zone:get_env(Zone, bypass_auth_plugins, false),
%% the `anonymous` field in auth result does not mean the client is
%% connected without username, but if the auth result is based on
%% allowing anonymous access.
IsResultBasedOnAllowAnonymous =
case AllowAnonymous of
true -> true;
_ -> false
end,
Result = case AllowAnonymous of
true -> #{auth_result => success, anonymous => IsResultBasedOnAllowAnonymous};
_ -> #{auth_result => not_authorized, anonymous => IsResultBasedOnAllowAnonymous}
end,
case {IsAnonymous, AllowAnonymous} of
{true, false_quick_deny} ->
{stop, Result};
_ when Bypass ->
{stop, Result};
_ ->
{continue, Result}
end.
-compile({inline, [return_auth_result/1]}).

4
src/emqx_shared_sub.erl
View File

@ -242,7 +242,7 @@ maybe_nack_dropped(Msg) ->
%% For fresh Ref we send a nack and return true, to note that the inflight is full
{Sender, {fresh, _Group, Ref}} -> nack(Sender, Ref, dropped), drop;
%% For retry Ref we can't reject a message if inflight is full, so we mark it as
%% acknowledged and put it into mqueue
{_Sender, {retry, _Group, _Ref}} -> maybe_ack(Msg), store;
@ -311,7 +311,7 @@ do_pick(Strategy, ClientId, SourceTopic, Group, Topic, FailedSubs) ->
false;
[] ->
%% We try redispatch to subs who dropped the message because inflight was full.
Found = maps_find_by(FailedSubs, fun({SubPid, FailReason}) ->
Found = maps_find_by(FailedSubs, fun(SubPid, FailReason) ->
FailReason == dropped andalso is_alive_sub(SubPid)
end),
case Found of

6
test/emqx_access_control_SUITE.erl
View File

@ -38,6 +38,12 @@ t_authenticate(_) ->
emqx_zone:set_env(zone, allow_anonymous, true),
?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
t_authenticate_fast_fail(_) ->
emqx_zone:set_env(zone, allow_anonymous, false_quick_deny),
?assertMatch({error, _}, emqx_access_control:authenticate(clientinfo())),
emqx_zone:set_env(zone, allow_anonymous, true),
?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())).
t_check_acl(_) ->
emqx_zone:set_env(zone, acl_nomatch, deny),
application:set_env(emqx, enable_acl_cache, false),