From 662f7438fa078c3da695687532e9180a5186ee21 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 1 Sep 2021 15:07:20 +0800 Subject: [PATCH 01/84] chore: dos2unix format find . -type f -print0 | xargs -0 dos2unix --- .../src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml | 2 +- apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml | 2 +- .../src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml | 2 +- apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml | 2 +- apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml | 2 +- apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml index 1f929cd98..c620a5e2a 100644 --- a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml +++ b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml @@ -1,4 +1,4 @@ - + accepted: pass 2nd review + +accepted: auto tag '#triage/accepted' +accepted: untag '#needs-triage' +accepted: issue is ready to be worked on +accepted: in backlog, need planning + +assigned: update type tag (#support | #bug | #feature) +assigned: tag '#triage/accepted' +assigned: untag '#new, new_waiting' +assigned: update assignee +assigned: update priority + +InProgress: Update with link to the PR +InProgress: Update release tag +InProgress: Patch testing with issue reporter + +needs_information: tag '#triage/needs-information', notify reporter + +stale: untag '#triage/wait' +stale: tag '#stale' and notify reporter + +closed: github close issue +closed: converted to discussion + +[*]--> new: created + +new --> accepted: pass 1st review +new --> closed: If the issue is a topic \nfor discussion(not for bug or support) +new --> new_waiting: lack of info + +new_waiting --> stale: 7 days no updates +stale ---> closed: 14 days no updates +stale ---> new_waiting: updated info +closed --> [*] + +accepted -down--> assigned: priority review + +accepted --> needs_information: need more information\n to proceeed +needs_information --> accepted: updates +assigned --> InProgress: In sprint run\n or\n start to work on +InProgress --> closed: issue is solved +InProgress --->InProgress: More info is required from issuer reporter +needs_information -----> stale: no updates \n after 14 days + +note left of new_waiting + next review: 5 days +end note + +note right of accepted + using priority tag + - #priority/critical-urgent + - #priority/important-soon + - #priority/important-longterm + - #priority/backlog + - #priority/awaiting-more-evidence + using area tag + - #area/lb + - #area/acl + - #area/config + ... +end note + +@enduml From ffded5ab30c0d5c34dc1355bbc27216ababfbc4d Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 Sep 2021 12:40:44 +0200 Subject: [PATCH 06/84] chore(github-issue): update issue templates --- .github/ISSUE_TEMPLATE/bug-report.md | 21 ++++++++++++++++---- .github/ISSUE_TEMPLATE/feature-request.md | 5 +++++ .github/ISSUE_TEMPLATE/support-needed.md | 24 ++++++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 3ec513a37..188d9c821 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -8,9 +8,15 @@ labels: "Support, needs-triage" + -**Environment**: +### Subject of the issue +Describe your issue here. +### Your environment - EMQ X version (e.g. `emqx_ctl status`): - Hardware configuration (e.g. `lscpu`): - OS (e.g. `cat /etc/os-release`): @@ -18,8 +24,15 @@ labels: "Support, needs-triage" - Erlang/OTP version (in case you build emqx from source code): - Others: -**What happened and what you expected to happen**: +### Steps to reproduce +Tell us how to reproduce this issue. -**How to reproduce it (as minimally and precisely as possible)**: +It would be great if you can attach the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) -**Anything else we need to know?**: +### Expected behaviour +Tell us what should happen + +### Actual behaviour +Tell us what happens instead + +Missing log file can delay the handling of the issue. diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 1fb5f401f..ce45b054a 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -7,6 +7,11 @@ labels: "Feature, needs-triage" --- + + **What would you like to be added/modified**: diff --git a/.github/ISSUE_TEMPLATE/support-needed.md b/.github/ISSUE_TEMPLATE/support-needed.md index a19299c42..68e59bdbc 100644 --- a/.github/ISSUE_TEMPLATE/support-needed.md +++ b/.github/ISSUE_TEMPLATE/support-needed.md @@ -6,4 +6,26 @@ labels: "Support, needs-triage" --- -**Please describe your problem in detail, if necessary, you can upload the log file through the attachment**: + + +### Subject of the support +Describe your issue here. + +Error/Warning printout if any. + +### Your environment +- EMQ X version (e.g. `emqx_ctl status`): +- If cluster (e.g. 3 X 4Core16GB): +- Hardware configuration (e.g. `lscpu`): +- OS (e.g. `cat /etc/os-release`): +- Kernel (e.g. `uname -a`): +- Erlang/OTP version (in case you build emqx from source code): +- Others: + +### LOG File +It would be great if you can attach the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) + +Missing log file can delay the handling of the issue. From 24aaa5349bbf5617ae6a6292cf7454c7c215b7fa Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 7 Sep 2021 10:00:19 +0800 Subject: [PATCH 07/84] chore(cluster-call): more clear function/var name --- apps/emqx_machine/src/emqx_cluster_rpc.erl | 144 +++++++++--------- .../test/emqx_cluster_rpc_SUITE.erl | 2 +- 2 files changed, 70 insertions(+), 76 deletions(-) diff --git a/apps/emqx_machine/src/emqx_cluster_rpc.erl b/apps/emqx_machine/src/emqx_cluster_rpc.erl index a73bd1e63..d5721c8c2 100644 --- a/apps/emqx_machine/src/emqx_cluster_rpc.erl +++ b/apps/emqx_machine/src/emqx_cluster_rpc.erl @@ -85,7 +85,7 @@ multicall(M, F, A) -> TnxId :: pos_integer(), Timeout :: timeout(), Reason :: string(). -multicall(M, F, A, SucceedNum, Timeout) -> +multicall(M, F, A, RequireNum, Timeout)when RequireNum >= 1 -> MFA = {initiate, {M, F, A}}, Begin = erlang:monotonic_time(), InitRes = @@ -101,9 +101,23 @@ multicall(M, F, A, SucceedNum, Timeout) -> end end, End = erlang:monotonic_time(), - Gap = erlang:convert_time_unit(Begin - End, native, millisecond) + 50, - RetryMs = get_retry_ms(), - confirm_commit(InitRes, SucceedNum, Gap, 3 * max(Gap, RetryMs)). + MinDelay = erlang:convert_time_unit(Begin - End, native, millisecond) + 50, + %% Failed after 3 attempts. + RetryTimeout = 3 * max(MinDelay, get_retry_ms()), + OkOrFailed = + case InitRes of + {ok, _TnxId, _} when RequireNum =:= 1 -> + ok; + {ok, TnxId, _} when RequireNum =:= all -> + wait_for_all_nodes_commit(TnxId, MinDelay, RetryTimeout); + {ok, TnxId, _} when is_integer(RequireNum) -> + wait_for_nodes_commit(RequireNum, TnxId, MinDelay, RetryTimeout); + Error -> Error + end, + case OkOrFailed of + ok -> InitRes; + _ -> OkOrFailed + end. -spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}. query(TnxId) -> @@ -294,83 +308,63 @@ trans_query(TnxId) -> -define(TO_BIN(_B_), iolist_to_binary(io_lib:format("~p", [_B_]))). -apply_mfa(TnxId, {M, F, A} = MFA) -> - Result = - try - Meta0 = #{tnx_id => TnxId, module => M, function => F, args => ?TO_BIN(A)}, - Res = erlang:apply(M, F, A), - Succeed = - case Res of - ok -> - OkMeta = Meta0#{msg => <<"succeeded to apply MFA">>, result => Res}, - ?SLOG(notice, OkMeta), - {true, OkMeta}; - {ok, _} -> - OkMeta1 = Meta0#{msg => <<"succeeded to apply MFA">>, result => ?TO_BIN(Res)}, - ?SLOG(notice, OkMeta1), - {true, OkMeta1}; - _ -> - NotOkMeta = Meta0#{msg => <<"failed to apply MFA">>, result => ?TO_BIN(Res)}, - ?SLOG(error, NotOkMeta), - {false, NotOkMeta} - end, - {Succeed, Res} +apply_mfa(TnxId, {M, F, A}) -> + Res = + try erlang:apply(M, F, A) catch - C : E -> - CrashMeta = #{msg => <<"crash to apply MFA">>, tnx_id => TnxId, exception => C, reason => ?TO_BIN(E), - module => M, function => F, args => ?TO_BIN(A)}, - ?SLOG(critical, CrashMeta), - {{false, CrashMeta}, lists:flatten(io_lib:format("TnxId(~p) apply MFA(~p) crash", [TnxId, MFA]))} - end, - case Result of - {{true, Meta}, OkRes} -> - emqx_alarm:deactivate(cluster_rpc_apply_failed, Meta), - {true, OkRes}; - {{false, Meta}, NotOkRes} -> - emqx_alarm:activate(cluster_rpc_apply_failed, Meta), - {false, NotOkRes} + C : E -> {crash, C, E} + end, + Meta = #{tnx_id => TnxId, module => M, function => F, args => ?TO_BIN(A)}, + log_and_alarm(Res, Meta), + Succeed = (Res =:= ok orelse (is_tuple(Res) andalso Res =/= {} andalso element(1, Res) =:= ok)), + {Succeed, Res}. + +log_and_alarm(ok, Meta) -> + OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => <<"ok">>}, + ?SLOG(notice, OkMeta), + emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta); +log_and_alarm({ok, _} = Res, Meta) -> + OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => ?TO_BIN(Res)}, + ?SLOG(notice, OkMeta), + emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta); +log_and_alarm({crash, C, E}, Meta) -> + CrashMeta = Meta#{msg => <<"crash to apply MFA">>, exception => C, reason => ?TO_BIN(E)}, + ?SLOG(critical, CrashMeta), + emqx_alarm:activate(cluster_rpc_apply_failed, CrashMeta); +log_and_alarm(Res, Meta) -> + NotOkMeta = Meta#{msg => <<"failed to apply MFA">>, result => ?TO_BIN(Res)}, + ?SLOG(error, NotOkMeta), + emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta). + +wait_for_all_nodes_commit(TnxId, Delay, Remain) -> + ok = timer:sleep(Delay), + case legging_node(TnxId) of + [_|_] when Remain > 0 -> + wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); + [] -> ok; + Nodes -> {error, Nodes} end. -confirm_commit({ok, TnxId, _Res} = Init, all, _Gap, Timeout) when Timeout =< 0 -> - case node_not_commit(TnxId) of - ok -> Init; - Error -> Error - end; -confirm_commit({ok, TnxId, _Res} = Init, all, Gap, Timeout) -> - timer:sleep(Gap), - case node_not_commit(TnxId) of - ok -> Init; - _Error -> confirm_commit(Init, all, Gap, Timeout - Gap) - end; -confirm_commit(Init, 1, _Gap, _Timeout) -> Init; -confirm_commit({ok, TnxId, _Res} = Init, SucceedNum, _Gap, Timeout) when Timeout =< 0 -> - case node_already_commit(TnxId, SucceedNum) of - ok -> Init; - _Error -> - case node_not_commit(TnxId) of - ok -> Init; %% All commit but The succeedNum is greater than the length(nodes()). - Error -> Error +wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> + ok = timer:sleep(Delay), + case length(synced_nodes(TnxId)) >= RequiredNum of + true -> ok; + false when Remain > 0 -> + wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain - Delay); + false -> + case legging_node(TnxId) of + [] -> ok; %% All commit but The succeedNum > length(nodes()). + Nodes -> {error, Nodes} end - end; -confirm_commit({ok, TnxId, _Res} = Init, SucceedNum, Gap, Timeout) -> - timer:sleep(Gap), - case node_already_commit(TnxId, SucceedNum) of - ok -> Init; - _Error -> confirm_commit(Init, SucceedNum, Gap, Timeout - Gap) - end; -confirm_commit(Error, _SucceedNum, _Gap, _Timeout) -> Error. - -node_not_commit(TnxId) -> - case transaction(fun commit_status_trans/2, ['<', TnxId]) of - {atomic, []} -> ok; - {atomic, Nodes} -> {error, Nodes} end. -node_already_commit(TnxId, SucceedNum) -> - case transaction(fun commit_status_trans/2, ['>=', TnxId]) of - {atomic, Nodes} when length(Nodes) >= SucceedNum -> ok; - {atomic, Nodes} -> {error, Nodes} - end. +legging_node(TnxId) -> + {atomic, Nodes} = transaction(fun commit_status_trans/2, ['<', TnxId]), + Nodes. + +synced_nodes(TnxId) -> + {atomic, Nodes} = transaction(fun commit_status_trans/2, ['>=', TnxId]), + Nodes. commit_status_trans(Operator, TnxId) -> MatchHead = #cluster_rpc_commit{tnx_id = '$1', node = '$2', _ = '_'}, diff --git a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl index d790c7069..ebb654af6 100644 --- a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl @@ -94,7 +94,7 @@ t_commit_crash_test(_Config) -> {atomic, []} = emqx_cluster_rpc:status(), {M, F, A} = {?MODULE, no_exist_function, []}, Error = emqx_cluster_rpc:multicall(M, F, A), - ?assertEqual({error, "TnxId(1) apply MFA({emqx_cluster_rpc_SUITE,no_exist_function,[]}) crash"}, Error), + ?assertEqual({error, {crash,error,undef}}, Error), ?assertEqual({atomic, []}, emqx_cluster_rpc:status()), ok. From 6bc7378e63f7b20933ec56fdcbac4754aca9e38a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 10 Sep 2021 09:40:30 +0800 Subject: [PATCH 08/84] fix(cluster-call): fix typo and add is_success/1 help function --- apps/emqx_machine/src/emqx_cluster_rpc.erl | 51 ++++++++++------------ 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/apps/emqx_machine/src/emqx_cluster_rpc.erl b/apps/emqx_machine/src/emqx_cluster_rpc.erl index d5721c8c2..c747f7654 100644 --- a/apps/emqx_machine/src/emqx_cluster_rpc.erl +++ b/apps/emqx_machine/src/emqx_cluster_rpc.erl @@ -85,7 +85,7 @@ multicall(M, F, A) -> TnxId :: pos_integer(), Timeout :: timeout(), Reason :: string(). -multicall(M, F, A, RequireNum, Timeout)when RequireNum >= 1 -> +multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNum >= 1 -> MFA = {initiate, {M, F, A}}, Begin = erlang:monotonic_time(), InitRes = @@ -102,7 +102,7 @@ multicall(M, F, A, RequireNum, Timeout)when RequireNum >= 1 -> end, End = erlang:monotonic_time(), MinDelay = erlang:convert_time_unit(Begin - End, native, millisecond) + 50, - %% Failed after 3 attempts. + %% Fail after 3 attempts. RetryTimeout = 3 * max(MinDelay, get_retry_ms()), OkOrFailed = case InitRes of @@ -191,7 +191,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) -> {atomic, caught_up} -> ?TIMEOUT; {atomic, {still_lagging, NextId, MFA}} -> {Succeed, _} = apply_mfa(NextId, MFA), - case Succeed orelse SkipResult of + case Succeed orelse SkipResult of true -> case transaction(fun commit/2, [Node, NextId]) of {atomic, ok} -> catch_up(State, false); @@ -312,34 +312,31 @@ apply_mfa(TnxId, {M, F, A}) -> Res = try erlang:apply(M, F, A) catch - C : E -> {crash, C, E} + Class:Reason:Stacktrace -> + {error, #{exception => Class, reason => Reason, stacktrace => Stacktrace}} end, Meta = #{tnx_id => TnxId, module => M, function => F, args => ?TO_BIN(A)}, - log_and_alarm(Res, Meta), - Succeed = (Res =:= ok orelse (is_tuple(Res) andalso Res =/= {} andalso element(1, Res) =:= ok)), - {Succeed, Res}. + IsSuccess = is_success(Res), + log_and_alarm(IsSuccess, Res, Meta), + {IsSuccess, Res}. -log_and_alarm(ok, Meta) -> - OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => <<"ok">>}, - ?SLOG(notice, OkMeta), - emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta); -log_and_alarm({ok, _} = Res, Meta) -> - OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => ?TO_BIN(Res)}, - ?SLOG(notice, OkMeta), - emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta); -log_and_alarm({crash, C, E}, Meta) -> - CrashMeta = Meta#{msg => <<"crash to apply MFA">>, exception => C, reason => ?TO_BIN(E)}, - ?SLOG(critical, CrashMeta), - emqx_alarm:activate(cluster_rpc_apply_failed, CrashMeta); -log_and_alarm(Res, Meta) -> - NotOkMeta = Meta#{msg => <<"failed to apply MFA">>, result => ?TO_BIN(Res)}, +is_success(ok) -> true; +is_success({ok, _}) -> true; +is_success(_) -> false. + +log_and_alarm(true, Res, Meta) -> + OkMeta = Meta#{msg => <<"succeeded to apply MFA">>, result => Res}, + ?SLOG(debug, OkMeta), + emqx_alarm:deactivate(cluster_rpc_apply_failed, OkMeta#{result => ?TO_BIN(Res)}); +log_and_alarm(false, Res, Meta) -> + NotOkMeta = Meta#{msg => <<"failed to apply MFA">>, result => Res}, ?SLOG(error, NotOkMeta), - emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta). + emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta#{result => ?TO_BIN(Res)}). wait_for_all_nodes_commit(TnxId, Delay, Remain) -> ok = timer:sleep(Delay), - case legging_node(TnxId) of - [_|_] when Remain > 0 -> + case lagging_node(TnxId) of + [_ | _] when Remain > 0 -> wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); [] -> ok; Nodes -> {error, Nodes} @@ -352,18 +349,18 @@ wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> false when Remain > 0 -> wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain - Delay); false -> - case legging_node(TnxId) of + case lagging_node(TnxId) of [] -> ok; %% All commit but The succeedNum > length(nodes()). Nodes -> {error, Nodes} end end. -legging_node(TnxId) -> +lagging_node(TnxId) -> {atomic, Nodes} = transaction(fun commit_status_trans/2, ['<', TnxId]), Nodes. synced_nodes(TnxId) -> - {atomic, Nodes} = transaction(fun commit_status_trans/2, ['>=', TnxId]), + {atomic, Nodes} = transaction(fun commit_status_trans/2, ['>=', TnxId]), Nodes. commit_status_trans(Operator, TnxId) -> From 9820027953cc81583c27d50565665a85fbee38a9 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 10 Sep 2021 13:17:17 +0800 Subject: [PATCH 09/84] fix(cluster-call): fixed crash test case --- apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl index ebb654af6..cb07db37e 100644 --- a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl @@ -93,8 +93,10 @@ t_commit_crash_test(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), {M, F, A} = {?MODULE, no_exist_function, []}, - Error = emqx_cluster_rpc:multicall(M, F, A), - ?assertEqual({error, {crash,error,undef}}, Error), + {error, {error, Meta}} = emqx_cluster_rpc:multicall(M, F, A), + ?assertEqual(undef, maps:get(reason, Meta)), + ?assertEqual(error, maps:get(exception, Meta)), + ?assertEqual(true, maps:is_key(stacktrace, Meta)), ?assertEqual({atomic, []}, emqx_cluster_rpc:status()), ok. From 0303e593c5e48c5e730b07c732a2ba5aa94373e7 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 10 Sep 2021 13:27:43 +0800 Subject: [PATCH 10/84] fix: undef function --- apps/emqx_rule_engine/src/emqx_rule_registry.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index f464f1e15..a0b8b48d5 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -223,7 +223,7 @@ remove_rules(Rules) -> insert_rules([]) -> ok; insert_rules(Rules) -> - _ = emqx_rule_utils:cluster_call(?MODULE, load_hooks_for_rule, [Rules]), + _ = emqx_plugin_libs_rule:cluster_call(?MODULE, load_hooks_for_rule, [Rules]), [mnesia:write(?RULE_TAB, Rule, write) ||Rule <- Rules]. %% @private @@ -241,7 +241,7 @@ delete_rules(Rules = [Rule|_]) when is_record(Rule, rule) -> delete_rules_unload_hooks(Rules). delete_rules_unload_hooks(Rules) -> - _ = emqx_rule_utils:cluster_call(?MODULE, unload_hooks_for_rule, [Rules]), + _ = emqx_plugin_libs_rule:cluster_call(?MODULE, unload_hooks_for_rule, [Rules]), [mnesia:delete_object(?RULE_TAB, Rule, write) ||Rule <- Rules]. load_hooks_for_rule(Rules) -> From 311be8896966c824f3075161837c2505d488c688 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 10 Sep 2021 10:08:30 +0200 Subject: [PATCH 11/84] chore(github-issue_flow): address some remarks --- .../{asserts => assets}/issue-handling.png | Bin .../{asserts => assets}/issue-handling.uml | 0 .github/ISSUE_TEMPLATE/bug-report.md | 9 +++++---- .github/ISSUE_TEMPLATE/feature-request.md | 2 +- .github/ISSUE_TEMPLATE/support-needed.md | 7 +++++-- 5 files changed, 11 insertions(+), 7 deletions(-) rename .github/ISSUE_TEMPLATE/{asserts => assets}/issue-handling.png (100%) rename .github/ISSUE_TEMPLATE/{asserts => assets}/issue-handling.uml (100%) diff --git a/.github/ISSUE_TEMPLATE/asserts/issue-handling.png b/.github/ISSUE_TEMPLATE/assets/issue-handling.png similarity index 100% rename from .github/ISSUE_TEMPLATE/asserts/issue-handling.png rename to .github/ISSUE_TEMPLATE/assets/issue-handling.png diff --git a/.github/ISSUE_TEMPLATE/asserts/issue-handling.uml b/.github/ISSUE_TEMPLATE/assets/issue-handling.uml similarity index 100% rename from .github/ISSUE_TEMPLATE/asserts/issue-handling.uml rename to .github/ISSUE_TEMPLATE/assets/issue-handling.uml diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 188d9c821..825992ce8 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -2,7 +2,7 @@ name: Bug Report about: Create a report to help us improve title: '' -labels: "Support, needs-triage" +labels: needs-triage --- @@ -10,13 +10,16 @@ labels: "Support, needs-triage" ### Subject of the issue Describe your issue here. ### Your environment + +For EMQ X 4.3 or newer, please provide the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) + - EMQ X version (e.g. `emqx_ctl status`): - Hardware configuration (e.g. `lscpu`): - OS (e.g. `cat /etc/os-release`): @@ -27,8 +30,6 @@ Describe your issue here. ### Steps to reproduce Tell us how to reproduce this issue. -It would be great if you can attach the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) - ### Expected behaviour Tell us what should happen diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index ce45b054a..2f55bcfb8 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -9,7 +9,7 @@ labels: "Feature, needs-triage" diff --git a/.github/ISSUE_TEMPLATE/support-needed.md b/.github/ISSUE_TEMPLATE/support-needed.md index 68e59bdbc..49ba5a913 100644 --- a/.github/ISSUE_TEMPLATE/support-needed.md +++ b/.github/ISSUE_TEMPLATE/support-needed.md @@ -8,7 +8,7 @@ labels: "Support, needs-triage" ### Subject of the support @@ -17,6 +17,10 @@ Describe your issue here. Error/Warning printout if any. ### Your environment + +For EMQ X 4.3 or newer, please provide the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) + +Otherwise please provide below info: - EMQ X version (e.g. `emqx_ctl status`): - If cluster (e.g. 3 X 4Core16GB): - Hardware configuration (e.g. `lscpu`): @@ -26,6 +30,5 @@ Error/Warning printout if any. - Others: ### LOG File -It would be great if you can attach the log archive generated by [node_dump tool](https://github.com/emqx/emqx/blob/master/bin/node_dump) Missing log file can delay the handling of the issue. From ef0bdf641a941374283bd720dcd1caabc27923f1 Mon Sep 17 00:00:00 2001 From: Alexander Babel Date: Tue, 7 Sep 2021 15:10:28 +0200 Subject: [PATCH 12/84] feat(helm): add support for networking.k8s.io/v1 --- deploy/charts/emqx/Chart.yaml | 2 +- deploy/charts/emqx/README.md | 2 ++ deploy/charts/emqx/templates/ingress.yaml | 30 ++++++++++++++++++++++- deploy/charts/emqx/values.yaml | 2 ++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index e9b8ae3e9..f76e0c1aa 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,7 +14,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 4.3.0 +version: 4.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 428999a44..d49ee692d 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -66,11 +66,13 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.externalIPs` | ExternalIPs for the service | [] | | `service.annotations` | Service annotations | {}(evaluated as a template)| | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | | `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | | `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | | `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | | `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | | `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | | `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | | `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index f527e41f4..7fc66a86a 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -1,5 +1,7 @@ {{- if .Values.ingress.dashboard.enabled -}} -{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -17,15 +19,28 @@ metadata: {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }} {{- end }} spec: +{{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }} +{{- end }} rules: {{- range $host := .Values.ingress.dashboard.hosts }} - host: {{ $host }} http: paths: - path: / + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: ImplementationSpecific + {{- end }} backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.dashboard }} + {{- else }} serviceName: {{ include "emqx.fullname" $ }} servicePort: {{ $.Values.service.dashboard }} + {{- end }} {{- end -}} {{- if .Values.ingress.dashboard.tls }} tls: @@ -52,15 +67,28 @@ metadata: {{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }} {{- end }} spec: +{{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }} +{{- end }} rules: {{- range $host := .Values.ingress.mgmt.hosts }} - host: {{ $host }} http: paths: - path: / + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: ImplementationSpecific + {{- end }} backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mgmt }} + {{- else }} serviceName: {{ include "emqx.fullname" $ }} servicePort: {{ $.Values.service.mgmt }} + {{- end }} {{- end -}} {{- if .Values.ingress.mgmt.tls }} tls: diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index bbf254e0a..72c8265c6 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -128,6 +128,7 @@ ingress: ## ingress for EMQX Dashboard dashboard: enabled: false + # ingressClassName: nginx annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" @@ -139,6 +140,7 @@ ingress: ## ingress for EMQX Mgmt API mgmt: enabled: false + # ingressClassName: nginx annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" From d443d26fcee2d269d4badbf8c7ce1775a5a736fb Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 13 Sep 2021 16:21:28 +0800 Subject: [PATCH 13/84] fix(auth mnesia api): fix api error for file type --- .github/workflows/run_api_tests.yaml | 6 +- apps/emqx_authz/src/emqx_authz.erl | 50 ++++++---- .../emqx_authz/src/emqx_authz_api_sources.erl | 95 ++++++++++--------- 3 files changed, 84 insertions(+), 67 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 15dd92f3b..57037616c 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -2,9 +2,9 @@ name: API Test Suite on: push: - tags: - - e* - - v* + tags: + - e* + - v* pull_request: jobs: diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 6fe2d7565..da776f291 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -87,15 +87,19 @@ pre_config_update({move, Type, <<"top">>}, Conf) when is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), NConf = [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2, - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({move, Type, <<"bottom">>}, Conf) when is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), NConf = lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)], - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) -> {Index1, _} = find_source_by_type(Type), @@ -107,8 +111,10 @@ pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Co NConf = lists:delete(Conf1, lists:droplast(List1)) ++ [Conf1] ++ [Conf2] ++ lists:delete(Conf1, List2), - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) -> {Index1, _} = find_source_by_type(Type), @@ -119,28 +125,38 @@ pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf NConf = lists:delete(Conf1, List1) ++ [Conf1] ++ lists:delete(Conf1, List2), - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({head, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Sources ++ Conf, - ok = check_dup_types(NConf), - {ok, Sources ++ Conf}; + case check_dup_types(NConf) of + ok -> {ok, Sources ++ Conf}; + Error -> Error + end; pre_config_update({tail, Sources}, Conf) when is_list(Sources), is_list(Conf) -> NConf = Conf ++ Sources, - ok = check_dup_types(NConf), - {ok, Conf ++ Sources}; + case check_dup_types(NConf) of + ok -> {ok, Conf ++ Sources}; + Error -> Error + end; pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), NConf = lists:droplast(List1) ++ [Source] ++ List2, - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) -> {_, Source} = find_source_by_type(Type), NConf = lists:delete(Source, Conf), - ok = check_dup_types(NConf), - {ok, NConf}; + case check_dup_types(NConf) of + ok -> {ok, NConf}; + Error -> Error + end; pre_config_update({_, Sources}, _Conf) when is_list(Sources)-> %% overwrite the entire config! {ok, Sources}. @@ -249,7 +265,7 @@ check_dup_types(Sources, [T0 | Tail]) -> end, 0, Sources) > 1 of true -> ?LOG(error, "The type is duplicated in the Authorization source"), - {error, authz_source_dup}; + {error, 'The type is duplicated in the Authorization source'}; false -> check_dup_types(Sources, Tail) end. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index a3f198a30..acef52f44 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -298,12 +298,20 @@ move_source_api() -> sources(get, _) -> Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) -> - {ok, Rules} = file:consult(Path), - lists:append(AccIn, [#{type => file, - enable => Enable, - rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], - annotations => #{status => healthy} - }]); + case file:consult(Path) of + {ok, Rules} -> + lists:append(AccIn, [#{type => file, + enable => Enable, + rules => iolist_to_binary([io_lib:format("~p.", [R]) || R <- Rules]), + annotations => #{status => healthy} + }]); + {error, _} -> + lists:append(AccIn, [#{type => file, + enable => Enable, + rules => <<"">>, + annotations => #{status => unhealthy} + }]) + end; (#{enable := false} = Source, AccIn) -> lists:append(AccIn, [Source#{annotations => #{status => unhealthy}}]); (#{type := _Type, annotations := #{id := Id}} = Source, AccIn) -> @@ -328,23 +336,14 @@ sources(get, _) -> lists:append(AccIn, [Source#{annotations => #{status => healthy}}]) end, [], emqx_authz:lookup()), {200, #{sources => Sources}}; -sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) when is_list(Rules) -> +sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) when is_list(Rules) -> {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), erlang:list_to_bitstring([<> || Rule <- Rules]) ), - case emqx_authz:update(head, [#{type => file, enable => Enable, path => Filename}]) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end; + + update_config(head, [#{type => file, enable => true, path => Filename}]); sources(post, #{body := Body}) when is_map(Body) -> - case emqx_authz:update(head, [write_cert(Body)]) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end; + update_config(head, [write_cert(Body)]); sources(put, #{body := Body}) when is_list(Body) -> NBody = [ begin case Source of @@ -354,24 +353,24 @@ sources(put, #{body := Body}) when is_list(Body) -> _ -> write_cert(Source) end end || Source <- Body], - case emqx_authz:update(replace, NBody) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end. + update_config(replace, NBody). source(get, #{bindings := #{type := Type}}) -> case emqx_authz:lookup(Type) of {error, Reason} -> {404, #{messgae => atom_to_binary(Reason)}}; #{type := file, enable := Enable, path := Path}-> - {ok, Rules} = file:consult(Path), - {200, #{type => file, - enable => Enable, - rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], - annotations => #{status => healthy} - } - }; + case file:consult(Path) of + {ok, Rules} -> + {200, #{type => file, + enable => Enable, + rules => iolist_to_binary([io_lib:format("~p.", [R]) || R <- Rules]), + annotations => #{status => healthy} + } + }; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; #{enable := false} = Source -> {200, Source#{annotations => #{status => unhealthy}}}; #{annotations := #{id := Id}} = Source -> NSource0 = case maps:get(server, Source, undefined) of @@ -401,22 +400,10 @@ source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file messgae => atom_to_binary(Reason)}} end; source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> - case emqx_authz:update({replace_once, Type}, write_cert(Body)) of - {ok, _} -> {204}; - {error, not_found_source} -> - {404, #{code => <<"NOT_FOUND">>, - messgae => <<"source ", Type/binary, " not found">>}}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end; + update_config({replace_once, Type}, write_cert(Body)); source(delete, #{bindings := #{type := Type}}) -> - case emqx_authz:update({delete_once, Type}, #{}) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end. + update_config({delete_once, Type}, #{}). + move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) -> case emqx_authz:move(Type, Position) of {ok, _} -> {204}; @@ -428,6 +415,20 @@ move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos messgae => atom_to_binary(Reason)}} end. +update_config(Cmd, Sources) -> + case emqx_authz:update(Cmd, Sources) of + {ok, _} -> {204}; + {error, {pre_config_update, emqx_authz, Reason}} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}}; + {error, {post_config_update, emqx_authz, Reason}} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end. + read_cert(#{ssl := #{enable := true} = SSL} = Source) -> CaCert = case file:read_file(maps:get(cacertfile, SSL, "")) of {ok, CaCert0} -> CaCert0; From 4fa816fa97f789588485d8d684ee1e3ce9e60b10 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 13 Sep 2021 18:16:23 +0800 Subject: [PATCH 14/84] fix(auth mnesia api): fix api error when delete sources --- apps/emqx_authz/src/emqx_authz.erl | 5 +++-- apps/emqx_authz/src/emqx_authz_api_sources.erl | 7 ++----- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index da776f291..9ba67ddd1 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -151,8 +151,9 @@ pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_ Error -> Error end; pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) -> - {_, Source} = find_source_by_type(Type), - NConf = lists:delete(Source, Conf), + {Index, _} = find_source_by_type(Type), + {List1, List2} = lists:split(Index, Conf), + NConf = lists:droplast(List1) ++ List2, case check_dup_types(NConf) of ok -> {ok, NConf}; Error -> Error diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index acef52f44..54ac28409 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -336,11 +336,8 @@ sources(get, _) -> lists:append(AccIn, [Source#{annotations => #{status => healthy}}]) end, [], emqx_authz:lookup()), {200, #{sources => Sources}}; -sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) when is_list(Rules) -> - {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), - erlang:list_to_bitstring([<> || Rule <- Rules]) - ), - +sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) -> + {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules), update_config(head, [#{type => file, enable => true, path => Filename}]); sources(post, #{body := Body}) when is_map(Body) -> update_config(head, [write_cert(Body)]); diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index b4b7b87c9..3862e06d0 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -219,6 +219,7 @@ t_api(_) -> end, Sources), {ok, 200, Result5} = request(get, uri(["authorization", "sources"]), []), ?assertEqual([], get_sources(Result5)), + ?assertEqual([], emqx:get_config([authorization, sources])), ok. t_move_source(_) -> From 99566f12bb5aa6f7bb9341ca33fa0ab90b8ee3ac Mon Sep 17 00:00:00 2001 From: lafirest Date: Fri, 10 Sep 2021 14:28:29 +0800 Subject: [PATCH 15/84] test(emqx_coap): add some test case --- .../src/coap/emqx_coap_channel.erl | 11 +- .../src/coap/test/emqx_coap_SUITE.erl | 319 -------- .../src/coap/test/emqx_coap_pubsub_SUITE.erl | 678 ------------------ apps/emqx_gateway/test/emqx_coap_SUITE.erl | 289 ++++++++ 4 files changed, 290 insertions(+), 1007 deletions(-) delete mode 100644 apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl delete mode 100644 apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl create mode 100644 apps/emqx_gateway/test/emqx_coap_SUITE.erl diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 112efdc44..86b1f023f 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -263,16 +263,7 @@ check_token(true, end; check_token(false, Msg, Channel) -> - case emqx_coap_message:get_option(uri_query, Msg) of - #{<<"clientid">> := _} -> - Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), - {ok, {outgoing, Reply}, Channel}; - #{<<"token">> := _} -> - Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), - {ok, {outgoing, Reply}, Channel}; - _ -> - call_session(handle_request, Msg, Channel) - end. + call_session(handle_request, Msg, Channel). try_takeover(idle, DesireId, Msg, Channel) -> case emqx_coap_message:get_option(uri_path, Msg, []) of diff --git a/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl deleted file mode 100644 index 3f55aa716..000000000 --- a/apps/emqx_gateway/src/coap/test/emqx_coap_SUITE.erl +++ /dev/null @@ -1,319 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_SUITE). - -% -compile(export_all). -% -compile(nowarn_export_all). - -% -include_lib("gen_coap/include/coap.hrl"). -% -include_lib("eunit/include/eunit.hrl"). -% -include_lib("emqx/include/emqx.hrl"). - -% -define(LOGT(Format, Args), ct:pal(Format, Args)). - -% all() -> emqx_ct:all(?MODULE). - -% init_per_suite(Config) -> -% emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), -% Config. - -% set_special_cfg(emqx_coap) -> -% Opts = application:get_env(emqx_coap, dtls_opts,[]), -% Opts2 = [{keyfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/key.pem")}, -% {certfile, emqx_ct_helpers:deps_path(emqx, "etc/certs/cert.pem")}], -% application:set_env(emqx_coap, dtls_opts, emqx_misc:merge_opts(Opts, Opts2)), -% application:set_env(emqx_coap, enable_stats, true); -% set_special_cfg(_) -> -% ok. - -% end_per_suite(Config) -> -% emqx_ct_helpers:stop_apps([emqx_coap]), -% Config. - -% %%-------------------------------------------------------------------- -% %% Test Cases -% %%-------------------------------------------------------------------- - -% t_publish(_Config) -> -% Topic = <<"abc">>, Payload = <<"123">>, -% TopicStr = binary_to_list(Topic), -% URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - -% %% Sub topic first -% emqx:subscribe(Topic), - -% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% {ok, changed, _} = Reply, - -% receive -% {deliver, Topic, Msg} -> -% ?assertEqual(Topic, Msg#message.topic), -% ?assertEqual(Payload, Msg#message.payload) -% after -% 500 -> -% ?assert(false) -% end. - -% t_publish_authz_deny(_Config) -> -% Topic = <<"abc">>, Payload = <<"123">>, -% TopicStr = binary_to_list(Topic), -% URI = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", - -% %% Sub topic first -% emqx:subscribe(Topic), - -% ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), -% ok = meck:expect(emqx_access_control, authorize, 3, deny), -% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?assertEqual({error,forbidden}, Reply), -% ok = meck:unload(emqx_access_control), -% receive -% {deliver, Topic, Msg} -> ct:fail({unexpected, {Topic, Msg}}) -% after -% 500 -> ok -% end. - -% t_observe(_Config) -> -% Topic = <<"abc">>, TopicStr = binary_to_list(Topic), -% Payload = <<"123">>, -% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", -% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), -% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - -% [SubPid] = emqx:subscribers(Topic), -% ?assert(is_pid(SubPid)), - -% %% Publish a message -% emqx:publish(emqx_message:make(Topic, Payload)), - -% Notif = receive_notification(), -% ?LOGT("observer get Notif=~p", [Notif]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, -% ?assertEqual(Payload, PayloadRecv), - -% er_coap_observer:stop(Pid), -% timer:sleep(100), - -% [] = emqx:subscribers(Topic). - -% t_observe_authz_deny(_Config) -> -% Topic = <<"abc">>, TopicStr = binary_to_list(Topic), -% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", -% ok = meck:new(emqx_access_control, [non_strict, passthrough, no_history]), -% ok = meck:expect(emqx_access_control, authorize, 3, deny), -% ?assertEqual({error,forbidden}, er_coap_observer:observe(Uri)), -% [] = emqx:subscribers(Topic), -% ok = meck:unload(emqx_access_control). - -% t_observe_wildcard(_Config) -> -% Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), -% Payload = <<"123">>, -% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", -% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), -% ?LOGT("observer Uri=~p, Pid=~p, N=~p, Code=~p, Content=~p", [Uri, Pid, N, Code, Content]), - -% [SubPid] = emqx:subscribers(Topic), -% ?assert(is_pid(SubPid)), - -% %% Publish a message -% emqx:publish(emqx_message:make(<<"a/b">>, Payload)), - -% Notif = receive_notification(), -% ?LOGT("observer get Notif=~p", [Notif]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, -% ?assertEqual(Payload, PayloadRecv), - -% er_coap_observer:stop(Pid), -% timer:sleep(100), - -% [] = emqx:subscribers(Topic). - -% t_observe_pub(_Config) -> -% Topic = <<"+/b">>, TopicStr = emqx_http_lib:uri_encode(binary_to_list(Topic)), -% Uri = "coap://127.0.0.1/mqtt/"++TopicStr++"?c=client1&u=tom&p=secret", -% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri), -% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - -% [SubPid] = emqx:subscribers(Topic), -% ?assert(is_pid(SubPid)), - -% Topic2 = <<"a/b">>, Payload2 = <<"UFO">>, -% TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), -% URI2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", - -% Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = Payload2}), -% {ok,changed, _} = Reply2, - -% Notif2 = receive_notification(), -% ?LOGT("observer get Notif2=~p", [Notif2]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, -% ?assertEqual(Payload2, PayloadRecv2), - -% Topic3 = <<"j/b">>, Payload3 = <<"ET629">>, -% TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), -% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=mike&p=guess", -% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), -% {ok,changed, _} = Reply3, - -% Notif3 = receive_notification(), -% ?LOGT("observer get Notif3=~p", [Notif3]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv3}} = Notif3, -% ?assertEqual(Payload3, PayloadRecv3), - -% er_coap_observer:stop(Pid). - -% t_one_clientid_sub_2_topics(_Config) -> -% Topic1 = <<"abc">>, TopicStr1 = binary_to_list(Topic1), -% Payload1 = <<"123">>, -% Uri1 = "coap://127.0.0.1/mqtt/"++TopicStr1++"?c=client1&u=tom&p=secret", -% {ok, Pid1, N1, Code1, Content1} = er_coap_observer:observe(Uri1), -% ?LOGT("observer 1 Pid=~p, N=~p, Code=~p, Content=~p", [Pid1, N1, Code1, Content1]), - -% [SubPid] = emqx:subscribers(Topic1), -% ?assert(is_pid(SubPid)), - -% Topic2 = <<"x/y">>, TopicStr2 = emqx_http_lib:uri_encode(binary_to_list(Topic2)), -% Payload2 = <<"456">>, -% Uri2 = "coap://127.0.0.1/mqtt/"++TopicStr2++"?c=client1&u=tom&p=secret", -% {ok, Pid2, N2, Code2, Content2} = er_coap_observer:observe(Uri2), -% ?LOGT("observer 2 Pid=~p, N=~p, Code=~p, Content=~p", [Pid2, N2, Code2, Content2]), - -% [SubPid] = emqx:subscribers(Topic2), -% ?assert(is_pid(SubPid)), - -% emqx:publish(emqx_message:make(Topic1, Payload1)), - -% Notif1 = receive_notification(), -% ?LOGT("observer 1 get Notif=~p", [Notif1]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv1}} = Notif1, -% ?assertEqual(Payload1, PayloadRecv1), - -% emqx:publish(emqx_message:make(Topic2, Payload2)), - -% Notif2 = receive_notification(), -% ?LOGT("observer 2 get Notif=~p", [Notif2]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv2}} = Notif2, -% ?assertEqual(Payload2, PayloadRecv2), - -% er_coap_observer:stop(Pid1), -% er_coap_observer:stop(Pid2). - -% t_invalid_parameter(_Config) -> -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% %% "cid=client2" is invaid -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, -% TopicStr3 = emqx_http_lib:uri_encode(binary_to_list(Topic3)), -% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?cid=client2&u=tom&p=simple", -% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), -% ?assertMatch({error,bad_request}, Reply3), - -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% %% "what=hello" is invaid -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% URI4 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?what=hello", -% Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), -% ?assertMatch({error, bad_request}, Reply4). - -% t_invalid_topic(_Config) -> -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% %% "a/b" is a valid topic string -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Topic3 = <<"a/b">>, Payload3 = <<"ET629">>, -% TopicStr3 = binary_to_list(Topic3), -% URI3 = "coap://127.0.0.1/mqtt/"++TopicStr3++"?c=client2&u=tom&p=simple", -% Reply3 = er_coap_client:request(put, URI3, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), -% ?assertMatch({ok,changed,_Content}, Reply3), - -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% %% "+?#" is invaid topic string -% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% URI4 = "coap://127.0.0.1/mqtt/"++"+?#"++"?what=hello", -% Reply4 = er_coap_client:request(put, URI4, #coap_content{format = <<"application/octet-stream">>, payload = Payload3}), -% ?assertMatch({error,bad_request}, Reply4). - -% % mqtt connection kicked by coap with same client id -% t_kick_1(_Config) -> -% URI = "coap://127.0.0.1/mqtt/abc?c=clientid&u=tom&p=secret", -% % workaround: emqx:subscribe does not kick same client id. -% spawn_monitor(fun() -> -% {ok, C} = emqtt:start_link([{host, "localhost"}, -% {clientid, <<"clientid">>}, -% {username, <<"plain">>}, -% {password, <<"plain">>}]), -% {ok, _} = emqtt:connect(C) end), -% er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, -% payload = <<"123">>}), -% receive -% {'DOWN', _, _, _, _} -> ok -% after 2000 -> -% ?assert(false) -% end. - -% % mqtt connection kicked by coap with same client id -% t_authz(_Config) -> -% OldPath = emqx:get_env(plugins_etc_dir), -% application:set_env(emqx, plugins_etc_dir, -% emqx_ct_helpers:deps_path(emqx_authz, "test")), -% Conf = #{<<"authz">> => -% #{<<"rules">> => -% [#{<<"principal">> =>#{<<"username">> => <<"coap">>}, -% <<"permission">> => deny, -% <<"topics">> => [<<"abc">>], -% <<"action">> => <<"publish">>} -% ]}}, -% ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)), -% application:ensure_all_started(emqx_authz), - -% emqx:subscribe(<<"abc">>), -% URI = "coap://127.0.0.1/mqtt/adbc?c=client1&u=coap&p=secret", -% er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, -% payload = <<"123">>}), -% receive -% _Something -> ?assert(false) -% after 2000 -> -% ok -% end, - -% ok = emqx_hooks:del('client.authorize', {emqx_authz, authorize}), -% file:delete(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf')), -% application:set_env(emqx, plugins_etc_dir, OldPath), -% application:stop(emqx_authz). - -% t_stats(_) -> -% ok. - -% t_auth_failure(_) -> -% ok. - -% t_qos_supprot(_) -> -% ok. - -% %%-------------------------------------------------------------------- -% %% Helpers - -% receive_notification() -> -% receive -% {coap_notify, Pid, N2, Code2, Content2} -> -% {coap_notify, Pid, N2, Code2, Content2} -% after 2000 -> -% receive_notification_timeout -% end. - -% testdir(DataPath) -> -% Ls = filename:split(DataPath), -% filename:join(lists:sublist(Ls, 1, length(Ls) - 1)). diff --git a/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl b/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl deleted file mode 100644 index 403cd8b2b..000000000 --- a/apps/emqx_gateway/src/coap/test/emqx_coap_pubsub_SUITE.erl +++ /dev/null @@ -1,678 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_pubsub_SUITE). - -% -compile(export_all). -% -compile(nowarn_export_all). - - -% -include_lib("gen_coap/include/coap.hrl"). -% -include_lib("eunit/include/eunit.hrl"). -% -include_lib("emqx/include/emqx.hrl"). - -% -define(LOGT(Format, Args), ct:pal(Format, Args)). - -% all() -> emqx_ct:all(?MODULE). - -% init_per_suite(Config) -> -% emqx_ct_helpers:start_apps([emqx_coap], fun set_special_cfg/1), -% Config. - -% set_special_cfg(emqx_coap) -> -% application:set_env(emqx_coap, enable_stats, true); -% set_special_cfg(_) -> -% ok. - -% end_per_suite(Config) -> -% emqx_ct_helpers:stop_apps([emqx_coap]), -% Config. - -% %%-------------------------------------------------------------------- -% %% Test Cases -% %%-------------------------------------------------------------------- - -% t_update_max_age(_Config) -> -% TopicInPayload = <<"topic1">>, -% Payload = <<";ct=42">>, -% Payload1 = <<";ct=50">>, -% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", -% URI2 = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% timer:sleep(50), - -% %% post to create the same topic but with different max age and ct value in payload -% Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 70, format = <<"application/link-format">>, payload = Payload1}), -% {ok,created, #coap_content{location_path = LocPath}} = Reply1, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{TopicInPayload, MaxAge2, CT2, _ResPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?assertEqual(70, MaxAge2), -% ?assertEqual(<<"50">>, CT2), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -% t_create_subtopic(_Config) -> -% TopicInPayload = <<"topic1">>, -% TopicInPayloadStr = "topic1", -% Payload = <<";ct=42">>, -% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", -% RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", - -% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% timer:sleep(50), - -% %% post to create the a sub topic -% SubPayload = <<";ct=42">>, -% SubTopicInPayloadStr = "subtopic", -% SubURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"?c=client1&u=tom&p=secret", -% SubRealURI = "coap://127.0.0.1/ps/"++TopicInPayloadStr++"/"++SubTopicInPayloadStr++"?c=client1&u=tom&p=secret", -% FullTopic = list_to_binary(TopicInPayloadStr++"/"++SubTopicInPayloadStr), -% Reply1 = er_coap_client:request(post, SubURI, #coap_content{format = <<"application/link-format">>, payload = SubPayload}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, -% ?assertEqual([<<"/ps/topic1/subtopic">>] ,LocPath1), -% [{FullTopic, MaxAge2, CT2, _ResPayload, _}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), -% ?assertEqual(60, MaxAge2), -% ?assertEqual(<<"42">>, CT2), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, SubRealURI), -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). - -% t_over_max_age(_Config) -> -% TopicInPayload = <<"topic1">>, -% Payload = <<";ct=42">>, -% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 2, format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(2, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% timer:sleep(3000), -% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)). - -% t_refreash_max_age(_Config) -> -% TopicInPayload = <<"topic1">>, -% Payload = <<";ct=42">>, -% Payload1 = <<";ct=50">>, -% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", -% RealURI = "coap://127.0.0.1/ps/topic1"++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% TopicInfo = [{TopicInPayload, MaxAge1, CT1, _ResPayload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?LOGT("TimeStamp=~p", [TimeStamp]), -% ?assertEqual(5, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% timer:sleep(3000), - -% %% post to create the same topic, the max age timer will be restarted with the new max age value -% Reply1 = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/link-format">>, payload = Payload1}), -% {ok,created, #coap_content{location_path = LocPath}} = Reply1, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{TopicInPayload, MaxAge2, CT2, _ResPayload, TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(TopicInPayload), -% ?LOGT("TimeStamp1=~p", [TimeStamp1]), -% ?assertEqual(5, MaxAge2), -% ?assertEqual(<<"50">>, CT2), - -% timer:sleep(3000), -% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_timeout(TopicInPayload)), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, RealURI). - -% t_case01_publish_post(_Config) -> -% timer:sleep(100), -% MainTopic = <<"maintopic">>, -% TopicInPayload = <<"topic1">>, -% Payload = <<";ct=42">>, -% MainTopicStr = binary_to_list(MainTopic), - -% %% post to create topic maintopic/topic1 -% URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", -% FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), -% Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, -% ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), -% [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT2), - -% %% post to publish message to topic maintopic/topic1 -% FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), -% URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", -% PubPayload = <<"PUBLISH">>, - -% %% Sub topic first -% emqx:subscribe(FullTopic), - -% Reply2 = er_coap_client:request(post, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), -% ?LOGT("Reply =~p", [Reply2]), -% {ok,changed, _} = Reply2, -% TopicInfo = [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), -% ?LOGT("the topic info =~p", [TopicInfo]), - -% assert_recv(FullTopic, PubPayload), -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -% t_case02_publish_post(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% Sub topic first -% emqx:subscribe(Topic), - -% %% post to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% assert_recv(Topic, Payload), - -% %% post to publish a new message to the same topic "topic1" with different payload -% NewPayload = <<"newpayload">>, -% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,changed, _} = Reply1, -% [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - -% assert_recv(Topic, NewPayload), -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case03_publish_post(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% Sub topic first -% emqx:subscribe(Topic), - -% %% post to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% assert_recv(Topic, Payload), - -% %% post to publish a new message to the same topic "topic1", but the ct is not same as created -% NewPayload = <<"newpayload">>, -% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), -% ?LOGT("Reply =~p", [Reply1]), -% ?assertEqual({error,bad_request}, Reply1), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case04_publish_post(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% post to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(5, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% %% after max age timeout, the topic still exists but the status is timeout -% timer:sleep(6000), -% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case01_publish_put(_Config) -> -% MainTopic = <<"maintopic">>, -% TopicInPayload = <<"topic1">>, -% Payload = <<";ct=42">>, -% MainTopicStr = binary_to_list(MainTopic), - -% %% post to create topic maintopic/topic1 -% URI1 = "coap://127.0.0.1/ps/"++MainTopicStr++"?c=client1&u=tom&p=secret", -% FullTopic = list_to_binary(MainTopicStr++"/"++binary_to_list(TopicInPayload)), -% Reply1 = er_coap_client:request(post, URI1, #coap_content{format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, -% ?assertEqual([<<"/ps/maintopic/topic1">>] ,LocPath1), -% [{FullTopic, MaxAge, CT2, <<>>, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT2), - -% %% put to publish message to topic maintopic/topic1 -% FullTopicStr = emqx_http_lib:uri_encode(binary_to_list(FullTopic)), -% URI2 = "coap://127.0.0.1/ps/"++FullTopicStr++"?c=client1&u=tom&p=secret", -% PubPayload = <<"PUBLISH">>, - -% %% Sub topic first -% emqx:subscribe(FullTopic), - -% Reply2 = er_coap_client:request(put, URI2, #coap_content{format = <<"application/octet-stream">>, payload = PubPayload}), -% ?LOGT("Reply =~p", [Reply2]), -% {ok,changed, _} = Reply2, -% [{FullTopic, MaxAge, CT2, PubPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(FullTopic), - -% assert_recv(FullTopic, PubPayload), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI2). - -% t_case02_publish_put(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% Sub topic first -% emqx:subscribe(Topic), - -% %% put to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% assert_recv(Topic, Payload), - -% %% put to publish a new message to the same topic "topic1" with different payload -% NewPayload = <<"newpayload">>, -% Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = NewPayload}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,changed, _} = Reply1, -% [{Topic, MaxAge, CT, NewPayload, _TimeStamp1}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), - -% assert_recv(Topic, NewPayload), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case03_publish_put(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% Sub topic first -% emqx:subscribe(Topic), - -% %% put to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(60, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% assert_recv(Topic, Payload), - -% %% put to publish a new message to the same topic "topic1", but the ct is not same as created -% NewPayload = <<"newpayload">>, -% Reply1 = er_coap_client:request(put, URI, #coap_content{format = <<"application/exi">>, payload = NewPayload}), -% ?LOGT("Reply =~p", [Reply1]), -% ?assertEqual({error,bad_request}, Reply1), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case04_publish_put(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"payload">>, - -% %% put to publish a new topic "topic1", and the topic is created -% URI = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(put, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/topic1">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(5, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% %% after max age timeout, no publish message to the same topic, the topic info will be deleted -% %%%%%%%%%%%%%%%%%%%%%%%%%% -% % but there is one thing to do is we don't count in the publish message received from emqx(from other node).TBD!!!!!!!!!!!!! -% %%%%%%%%%%%%%%%%%%%%%%%%%% -% timer:sleep(6000), -% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case01_subscribe(_Config) -> -% Topic = <<"topic1">>, -% Payload1 = <<";ct=42">>, -% timer:sleep(100), - -% %% First post to create a topic "topic1" -% Uri = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/link-format">>, payload = Payload1}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, -% ?assertEqual(<<"/ps/topic1">> ,LocPath), -% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% %% Subscribe the topic -% Uri1 = "coap://127.0.0.1"++binary_to_list(LocPath)++"?c=client1&u=tom&p=secret", -% {ok, Pid, N, Code, Content} = er_coap_observer:observe(Uri1), -% ?LOGT("observer Pid=~p, N=~p, Code=~p, Content=~p", [Pid, N, Code, Content]), - -% [SubPid] = emqx:subscribers(Topic), -% ?assert(is_pid(SubPid)), - -% %% Publish a message -% Payload = <<"123">>, -% emqx:publish(emqx_message:make(Topic, Payload)), - -% Notif = receive_notification(), -% ?LOGT("observer get Notif=~p", [Notif]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = PayloadRecv}} = Notif, - -% ?assertEqual(Payload, PayloadRecv), - -% %% GET to read the publish message of the topic -% Reply1 = er_coap_client:request(get, Uri1), -% ?LOGT("Reply=~p", [Reply1]), -% {ok,content, #coap_content{payload = <<"123">>}} = Reply1, - -% er_coap_observer:stop(Pid), -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri1). - -% t_case02_subscribe(_Config) -> -% Topic = <<"a/b">>, -% TopicStr = binary_to_list(Topic), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), -% Payload = <<"payload">>, - -% %% post to publish a new topic "a/b", and the topic is created -% URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/a/b">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(5, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% %% Wait for the max age of the timer expires -% timer:sleep(6000), -% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - -% %% Subscribe to the timeout topic "a/b", still successfully，got {ok, nocontent} Method -% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% Reply1 = {ok, Pid, _N, nocontent, _} = er_coap_observer:observe(Uri), -% ?LOGT("Subscribe Reply=~p", [Reply1]), - -% [SubPid] = emqx:subscribers(Topic), -% ?assert(is_pid(SubPid)), - -% %% put to publish to topic "a/b" -% Reply2 = er_coap_client:request(put, URI, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% {ok,changed, #coap_content{}} = Reply2, -% [{Topic, MaxAge1, CT, Payload, TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT), -% ?assertEqual(false, TimeStamp =:= timeout), - -% %% Publish a message -% emqx:publish(emqx_message:make(Topic, Payload)), - -% Notif = receive_notification(), -% ?LOGT("observer get Notif=~p", [Notif]), -% {coap_notify, _, _, {ok,content}, #coap_content{payload = Payload}} = Notif, - -% er_coap_observer:stop(Pid), -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case03_subscribe(_Config) -> -% %% Subscribe to the unexisted topic "a/b", got not_found -% Topic = <<"a/b">>, -% TopicStr = binary_to_list(Topic), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), -% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% {error, not_found} = er_coap_observer:observe(Uri), - -% [] = emqx:subscribers(Topic). - -% t_case04_subscribe(_Config) -> -% %% Subscribe to the wildcad topic "+/b", got bad_request -% Topic = <<"+/b">>, -% TopicStr = binary_to_list(Topic), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), -% Uri = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% {error, bad_request} = er_coap_observer:observe(Uri), - -% [] = emqx:subscribers(Topic). - -% t_case01_read(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"PubPayload">>, -% timer:sleep(100), - -% %% First post to create a topic "topic1" -% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, -% ?assertEqual(<<"/ps/topic1">> ,LocPath), -% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% %% GET to read the publish message of the topic -% timer:sleep(1000), -% Reply1 = er_coap_client:request(get, Uri), -% ?LOGT("Reply=~p", [Reply1]), -% {ok,content, #coap_content{payload = Payload}} = Reply1, - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -% t_case02_read(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"PubPayload">>, -% timer:sleep(100), - -% %% First post to publish a topic "topic1" -% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, -% ?assertEqual(<<"/ps/topic1">> ,LocPath), -% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% %% GET to read the publish message of unmatched format, got bad_request -% Reply1 = er_coap_client:request(get, Uri, #coap_content{format = <<"application/json">>}), -% ?LOGT("Reply=~p", [Reply1]), -% {error, bad_request} = Reply1, - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -% t_case03_read(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% timer:sleep(100), - -% %% GET to read the nexisted topic "topic1", got not_found -% Reply = er_coap_client:request(get, Uri), -% ?LOGT("Reply=~p", [Reply]), -% {error, not_found} = Reply. - -% t_case04_read(_Config) -> -% Topic = <<"topic1">>, -% TopicStr = binary_to_list(Topic), -% Payload = <<"PubPayload">>, -% timer:sleep(100), - -% %% First post to publish a topic "topic1" -% Uri = "coap://127.0.0.1/ps/"++TopicStr++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, Uri, #coap_content{format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = [LocPath]}} = Reply, -% ?assertEqual(<<"/ps/topic1">> ,LocPath), -% TopicInfo = [{Topic, MaxAge1, CT1, _ResPayload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?LOGT("lookup topic info=~p", [TopicInfo]), -% ?assertEqual(60, MaxAge1), -% ?assertEqual(<<"42">>, CT1), - -% %% GET to read the publish message of wildcard topic, got bad_request -% WildTopic = binary_to_list(<<"+/topic1">>), -% Uri1 = "coap://127.0.0.1/ps/"++WildTopic++"?c=client1&u=tom&p=secret", -% Reply1 = er_coap_client:request(get, Uri1, #coap_content{format = <<"application/json">>}), -% ?LOGT("Reply=~p", [Reply1]), -% {error, bad_request} = Reply1, - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, Uri). - -% t_case05_read(_Config) -> -% Topic = <<"a/b">>, -% TopicStr = binary_to_list(Topic), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), -% Payload = <<"payload">>, - -% %% post to publish a new topic "a/b", and the topic is created -% URI = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% Reply = er_coap_client:request(post, URI, #coap_content{max_age = 5, format = <<"application/octet-stream">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/a/b">>] ,LocPath), -% [{Topic, MaxAge, CT, Payload, _TimeStamp}] = emqx_coap_pubsub_topics:lookup_topic_info(Topic), -% ?assertEqual(5, MaxAge), -% ?assertEqual(<<"42">>, CT), - -% %% Wait for the max age of the timer expires -% timer:sleep(6000), -% ?assertEqual(true, emqx_coap_pubsub_topics:is_topic_timeout(Topic)), - -% %% GET to read the expired publish message, supposed to get {ok, nocontent}, but now got {ok, content} -% Reply1 = er_coap_client:request(get, URI), -% ?LOGT("Reply=~p", [Reply1]), -% {ok, content, #coap_content{payload = <<>>}}= Reply1, - -% {ok, deleted, #coap_content{}} = er_coap_client:request(delete, URI). - -% t_case01_delete(_Config) -> -% TopicInPayload = <<"a/b">>, -% TopicStr = binary_to_list(TopicInPayload), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), -% Payload = list_to_binary("<"++PercentEncodedTopic++">;ct=42"), -% URI = "coap://127.0.0.1/ps/"++"?c=client1&u=tom&p=secret", - -% %% Client post to CREATE topic "a/b" -% Reply = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload}), -% ?LOGT("Reply =~p", [Reply]), -% {ok,created, #coap_content{location_path = LocPath}} = Reply, -% ?assertEqual([<<"/ps/a/b">>] ,LocPath), - -% %% Client post to CREATE topic "a/b/c" -% TopicInPayload1 = <<"a/b/c">>, -% PercentEncodedTopic1 = emqx_http_lib:uri_encode(binary_to_list(TopicInPayload1)), -% Payload1 = list_to_binary("<"++PercentEncodedTopic1++">;ct=42"), -% Reply1 = er_coap_client:request(post, URI, #coap_content{format = <<"application/link-format">>, payload = Payload1}), -% ?LOGT("Reply =~p", [Reply1]), -% {ok,created, #coap_content{location_path = LocPath1}} = Reply1, -% ?assertEqual([<<"/ps/a/b/c">>] ,LocPath1), - -% timer:sleep(50), - -% %% DELETE the topic "a/b" -% UriD = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% ReplyD = er_coap_client:request(delete, UriD), -% ?LOGT("Reply=~p", [ReplyD]), -% {ok, deleted, #coap_content{}}= ReplyD, - -% timer:sleep(300), %% Waiting gen_server:cast/2 for deleting operation -% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload)), -% ?assertEqual(false, emqx_coap_pubsub_topics:is_topic_existed(TopicInPayload1)). - -% t_case02_delete(_Config) -> -% TopicInPayload = <<"a/b">>, -% TopicStr = binary_to_list(TopicInPayload), -% PercentEncodedTopic = emqx_http_lib:uri_encode(TopicStr), - -% %% DELETE the unexisted topic "a/b" -% Uri1 = "coap://127.0.0.1/ps/"++PercentEncodedTopic++"?c=client1&u=tom&p=secret", -% Reply1 = er_coap_client:request(delete, Uri1), -% ?LOGT("Reply=~p", [Reply1]), -% {error, not_found} = Reply1. - -% t_case13_emit_stats_test(_Config) -> -% ok. - -% %%-------------------------------------------------------------------- -% %% Internal functions - -% receive_notification() -> -% receive -% {coap_notify, Pid, N2, Code2, Content2} -> -% {coap_notify, Pid, N2, Code2, Content2} -% after 2000 -> -% receive_notification_timeout -% end. - -% assert_recv(Topic, Payload) -> -% receive -% {deliver, _, Msg} -> -% ?assertEqual(Topic, Msg#message.topic), -% ?assertEqual(Payload, Msg#message.payload) -% after -% 500 -> -% ?assert(false) -% end. - diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_SUITE.erl new file mode 100644 index 000000000..8e7352a74 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_coap_SUITE.erl @@ -0,0 +1,289 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("er_coap_client/include/coap.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<" +gateway.coap +{ + idle_timeout = 30s + enable_stats = false + mountpoint = \"\" + notify_type = qos + connection_required = true + subscribe_qos = qos1 + publish_qos = qos1 + authentication = undefined + + listeners.udp.default + {bind = 5683} +} +">>). + +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). +-define(PS_PREFIX, "coap://127.0.0.1/ps"). +-define(MQTT_PREFIX, "coap://127.0.0.1/mqtt"). + + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), + Config. + +set_special_cfg(emqx_gateway) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT); + +set_special_cfg(_) -> + ok. + +end_per_suite(Config) -> + emqx_ct_helpers:stop_apps([emqx_gateway]), + Config. + +%%-------------------------------------------------------------------- +%% Test Cases +%%-------------------------------------------------------------------- +t_connection(_Config) -> + Action = fun(Channel) -> + %% connection + Token = connection(Channel), + + timer:sleep(100), + ?assertNotEqual([], emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)), + + %% heartbeat + HeartURI = ?MQTT_PREFIX ++ "/connection?clientid=client1&token=" ++ Token, + ?LOGT("send heartbeat request:~s~n", [HeartURI]), + {ok, changed, _} = er_coap_client:request(put, HeartURI), + + disconnection(Channel, Token), + + timer:sleep(100), + ?assertEqual([], emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)) + end, + do(Action). + + +t_publish(_Config) -> + Action = fun(Channel, Token) -> + Topic = <<"/abc">>, + Payload = <<"123">>, + + TopicStr = binary_to_list(Topic), + URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, + + %% Sub topic first + emqx:subscribe(Topic), + + Req = make_req(post, Payload), + {ok, changed, _} = do_request(Channel, URI, Req), + + receive + {deliver, Topic, Msg} -> + ?assertEqual(Topic, Msg#message.topic), + ?assertEqual(Payload, Msg#message.payload) + after + 500 -> + ?assert(false) + end + end, + + with_connection(Action). + + +t_publish_authz_deny(_Config) -> + Action = fun(Channel, Token) -> + Topic = <<"/abc">>, + Payload = <<"123">>, + InvalidToken = lists:reverse(Token), + + TopicStr = binary_to_list(Topic), + URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ InvalidToken, + + %% Sub topic first + emqx:subscribe(Topic), + + Req = make_req(post, Payload), + Result = do_request(Channel, URI, Req), + ?assertEqual({error, reset}, Result) + end, + + with_connection(Action). + +t_subscribe(_Config) -> + Topic = <<"/abc">>, + Fun = fun(Channel, Token) -> + TopicStr = binary_to_list(Topic), + Payload = <<"123">>, + + URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, + Req = make_req(get, Payload, [{observe, 0}]), + {ok, content, _} = do_request(Channel, URI, Req), + ?LOGT("observer topic:~s~n", [Topic]), + + timer:sleep(100), + [SubPid] = emqx:subscribers(Topic), + ?assert(is_pid(SubPid)), + + %% Publish a message + emqx:publish(emqx_message:make(Topic, Payload)), + {ok, content, Notify} = with_response(Channel), + ?LOGT("observer get Notif=~p", [Notify]), + + #coap_content{payload = PayloadRecv} = Notify, + + ?assertEqual(Payload, PayloadRecv) + end, + + with_connection(Fun), + timer:sleep(100), + + ?assertEqual([], emqx:subscribers(Topic)). + + +t_un_subscribe(_Config) -> + Topic = <<"/abc">>, + Fun = fun(Channel, Token) -> + TopicStr = binary_to_list(Topic), + Payload = <<"123">>, + + URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, + + Req = make_req(get, Payload, [{observe, 0}]), + {ok, content, _} = do_request(Channel, URI, Req), + ?LOGT("observer topic:~s~n", [Topic]), + + timer:sleep(100), + [SubPid] = emqx:subscribers(Topic), + ?assert(is_pid(SubPid)), + + UnReq = make_req(get, Payload, [{observe, 1}]), + {ok, nocontent, _} = do_request(Channel, URI, UnReq), + ?LOGT("un observer topic:~s~n", [Topic]), + timer:sleep(100), + ?assertEqual([], emqx:subscribers(Topic)) + end, + + with_connection(Fun). + +t_observe_wildcard(_Config) -> + Fun = fun(Channel, Token) -> + %% resolve_url can't process wildcard with # + Topic = <<"/abc/+">>, + TopicStr = binary_to_list(Topic), + Payload = <<"123">>, + + URI = ?PS_PREFIX ++ TopicStr ++ "?clientid=client1&token=" ++ Token, + Req = make_req(get, Payload, [{observe, 0}]), + {ok, content, _} = do_request(Channel, URI, Req), + ?LOGT("observer topic:~s~n", [Topic]), + + timer:sleep(100), + [SubPid] = emqx:subscribers(Topic), + ?assert(is_pid(SubPid)), + + %% Publish a message + PubTopic = <<"/abc/def">>, + emqx:publish(emqx_message:make(PubTopic, Payload)), + {ok, content, Notify} = with_response(Channel), + + ?LOGT("observer get Notif=~p", [Notify]), + + #coap_content{payload = PayloadRecv} = Notify, + + ?assertEqual(Payload, PayloadRecv) + end, + + with_connection(Fun). + +connection(Channel) -> + URI = ?MQTT_PREFIX ++ "/connection?clientid=client1&username=admin&password=public", + Req = make_req(post), + {ok, created, Data} = do_request(Channel, URI, Req), + #coap_content{payload = BinToken} = Data, + binary_to_list(BinToken). + +disconnection(Channel, Token) -> + %% delete + URI = ?MQTT_PREFIX ++ "/connection?clientid=client1&token=" ++ Token, + Req = make_req(delete), + {ok, deleted, _} = do_request(Channel, URI, Req). + +make_req(Method) -> + make_req(Method, <<>>). + +make_req(Method, Payload) -> + make_req(Method, Payload, []). + +make_req(Method, Payload, Opts) -> + er_coap_message:request(con, Method, Payload, Opts). + +do_request(Channel, URI, #coap_message{options = Opts} = Req) -> + + {_, _, Path, Query} = er_coap_client:resolve_uri(URI), + Opts2 = [{uri_path, Path}, {uri_query, Query} | Opts], + Req2 = Req#coap_message{options = Opts2}, + ?LOGT("send request:~s~nReq:~p~n", [URI, Req2]), + + {ok, _} = er_coap_channel:send(Channel, Req2), + with_response(Channel). + +with_response(Channel) -> + receive + {coap_response, _ChId, Channel, _Ref, Message=#coap_message{method=Code}} -> + return_response(Code, Message); + {coap_error, _ChId, Channel, _Ref, reset} -> + {error, reset} + after 2000 -> + {error, timeout} + end. + +return_response({ok, Code}, Message) -> + {ok, Code, er_coap_message:get_content(Message)}; +return_response({error, Code}, #coap_message{payload= <<>>}) -> + {error, Code}; +return_response({error, Code}, Message) -> + {error, Code, er_coap_message:get_content(Message)}. + +do(Fun) -> + ChId = {{127, 0, 0, 1}, 5683}, + {ok, Sock} = er_coap_udp_socket:start_link(), + {ok, Channel} = er_coap_udp_socket:get_channel(Sock, ChId), + %% send and receive + Res = Fun(Channel), + %% terminate the processes + er_coap_channel:close(Channel), + er_coap_udp_socket:close(Sock), + Res. + +with_connection(Action) -> + Fun = fun(Channel) -> + Token = connection(Channel), + timer:sleep(100), + Action(Channel, Token), + disconnection(Channel, Token), + timer:sleep(100) + end, + do(Fun). From c3229572570a3431dbb0e8f9c0e78e2f52d9e426 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 14 Sep 2021 10:39:02 +0800 Subject: [PATCH 16/84] chore(authz): fix spelling errors --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 54ac28409..19d229dc2 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -354,7 +354,7 @@ sources(put, #{body := Body}) when is_list(Body) -> source(get, #{bindings := #{type := Type}}) -> case emqx_authz:lookup(Type) of - {error, Reason} -> {404, #{messgae => atom_to_binary(Reason)}}; + {error, Reason} -> {404, #{message => atom_to_binary(Reason)}}; #{type := file, enable := Enable, path := Path}-> case file:consult(Path) of {ok, Rules} -> @@ -366,7 +366,7 @@ source(get, #{bindings := #{type := Type}}) -> }; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} + message => atom_to_binary(Reason)}} end; #{enable := false} = Source -> {200, Source#{annotations => #{status => unhealthy}}}; #{annotations := #{id := Id}} = Source -> @@ -394,7 +394,7 @@ source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file {ok, _} -> {204}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} + message => atom_to_binary(Reason)}} end; source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> update_config({replace_once, Type}, write_cert(Body)); @@ -406,10 +406,10 @@ move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos {ok, _} -> {204}; {error, not_found_source} -> {404, #{code => <<"NOT_FOUND">>, - messgae => <<"source ", Type/binary, " not found">>}}; + message => <<"source ", Type/binary, " not found">>}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} + message => atom_to_binary(Reason)}} end. update_config(Cmd, Sources) -> @@ -417,13 +417,13 @@ update_config(Cmd, Sources) -> {ok, _} -> {204}; {error, {pre_config_update, emqx_authz, Reason}} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}}; + message => atom_to_binary(Reason)}}; {error, {post_config_update, emqx_authz, Reason}} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}}; + message => atom_to_binary(Reason)}}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} + message => atom_to_binary(Reason)}} end. read_cert(#{ssl := #{enable := true} = SSL} = Source) -> From acbbbbcd24db90776ef80cf12eeeef59241d980f Mon Sep 17 00:00:00 2001 From: lafirest Date: Mon, 13 Sep 2021 18:34:54 +0800 Subject: [PATCH 17/84] test(emqx_lwm2m): remove the lwm2m_coap dependency --- apps/emqx_gateway/rebar.config | 1 - apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl | 87 +++++++------ .../test/emqx_lwm2m_api_SUITE.erl | 114 ++---------------- 4 files changed, 52 insertions(+), 152 deletions(-) diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 8a0ad51e8..fe088d7d8 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. {deps, [ - {lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v2.0.0"}}}, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.2"}}} ]}. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 2fc329711..ca02511e3 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -3,7 +3,7 @@ {vsn, "0.1.0"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, lwm2m_coap, emqx]}, + {applications, [kernel, stdlib, grpc, emqx]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index 28edda7ef..2ee2312df 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -24,7 +24,7 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). -include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("lwm2m_coap/include/coap.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -50,6 +50,8 @@ gateway.lwm2m { } ">>). +-record(coap_content, {content_format, payload = <<>>}). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -199,7 +201,7 @@ case01_register(Config) -> ack = Type, {ok, created} = Method, RspId = MsgId, - Location = proplists:get_value(location_path, Opts), + Location = maps:get(location_path, Opts), ?assertNotEqual(undefined, Location), %% checkpoint 2 - verify subscribed topics @@ -247,7 +249,7 @@ case01_register_additional_opts(Config) -> Type = ack, Method = {ok, created}, RspId = MsgId, - Location = proplists:get_value(location_path, Opts), + Location = maps:get(location_path, Opts), ?assertNotEqual(undefined, Location), %% checkpoint 2 - verify subscribed topics @@ -318,14 +320,14 @@ case01_register_report(Config) -> Type = ack, Method = {ok, created}, RspId = MsgId, - Location = proplists:get_value(location_path, Opts), + Location = maps:get(location_path, Opts), ?assertNotEqual(undefined, Location), timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), ReadResult = emqx_json:encode(#{ - <<"msgType">> => <<"register">>, + <<"msgType">> => <<"register">>, <<"data">> => #{ <<"alternatePath">> => <<"/">>, <<"ep">> => list_to_binary(Epn), @@ -376,7 +378,7 @@ case02_update_deregister(Config) -> ?assertEqual({ok,created}, Method), ?LOGT("Options got: ~p", [Opts]), - Location = proplists:get_value(location_path, Opts), + Location = maps:get(location_path, Opts), Register = emqx_json:encode(#{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -1334,7 +1336,7 @@ case31_execute_error(Config) -> ?assertEqual(<<"2,7">>, Payload2), timer:sleep(50), - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, uauthorized}, #coap_content{}, Request2, true), + test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {error, unauthorized}, #coap_content{}, Request2, true), timer:sleep(100), ReadResult = emqx_json:encode(#{ @@ -1432,10 +1434,10 @@ case50_write_attribute(Config) -> #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, ?LOGT("got options: ~p", [Options2]), Path2 = get_coap_path(Options2), - Query2 = lists:sort(get_coap_query(Options2)), + Query2 = lists:sort(maps:to_list(get_coap_query(Options2))), ?assertEqual(put, Method2), ?assertEqual(<<"/3/0/9">>, Path2), - ?assertEqual(lists:sort([<<"pmax=5">>,<<"lt=5">>,<<"pmin=1">>]), Query2), + ?assertEqual(lists:sort([{<<"pmax">>, <<"5">>},{<<"lt">>, <<"5">>},{<<"pmin">>,<<"1">>}]), Query2), ?assertEqual(<<>>, Payload2), timer:sleep(50), @@ -1446,7 +1448,7 @@ case50_write_attribute(Config) -> #coap_content{}, Request2, true), - timer:sleep(100), + timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1742,7 +1744,7 @@ server_cache_mode(Config, RegOption) -> #coap_message{type = ack, method = Method1, options = Opts} = test_recv_coap_response(UdpSock), ?assertEqual({ok,created}, Method1), ?LOGT("Options got: ~p", [Opts]), - Location = proplists:get_value(location_path, Opts), + Location = maps:get(location_path, Opts), test_recv_mqtt_response(RespTopic), %% server not in PSM mode @@ -1836,10 +1838,10 @@ test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> is_list(Options) orelse error("Options must be a list"), case resolve_uri(Uri) of {coap, {IpAddr, Port}, Path, Query} -> - Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), + Request0 = request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), Request = Request0#coap_message{id = MsgId}, ?LOGT("send_coap_request Request=~p", [Request]), - RequestBinary = lwm2m_coap_message_parser:encode(Request), + RequestBinary = emqx_coap_frame:serialize_pkt(Request, undefined), ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); {SchemeDiff, ChIdDiff, _, _} -> @@ -1848,7 +1850,7 @@ test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> test_recv_coap_response(UdpSock) -> {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), - Response = lwm2m_coap_message_parser:decode(Packet), + {ok, Response, _, _} = emqx_coap_frame:parse(Packet, undefined), ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), @@ -1857,7 +1859,7 @@ test_recv_coap_response(UdpSock) -> test_recv_coap_request(UdpSock) -> case gen_udp:recv(UdpSock, 0, 2000) of {ok, {_Address, _Port, Packet}} -> - Request = lwm2m_coap_message_parser:decode(Packet), + {ok, Request, _, _} = emqx_coap_frame:parse(Packet, undefined), #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), Request; @@ -1871,32 +1873,32 @@ test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) -> is_list(Host) orelse error("Host is not a string"), {ok, IpAddr} = inet:getaddr(Host, inet), - Response = lwm2m_coap_message:response(Code, Content, Request), + Response = response(Code, Content, Request), Response2 = case Ack of true -> Response#coap_message{type = ack}; false -> Response end, ?LOGT("test_send_coap_response Response=~p", [Response2]), - ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)). + ok = gen_udp:send(UdpSock, IpAddr, Port, emqx_coap_frame:serialize_pkt(Response2, undefined)). test_send_empty_ack(UdpSock, Host, Port, Request) -> is_list(Host) orelse error("Host is not a string"), {ok, IpAddr} = inet:getaddr(Host, inet), - EmptyACK = lwm2m_coap_message:ack(Request), + EmptyACK = emqx_coap_message:ack(Request), ?LOGT("test_send_empty_ack EmptyACK=~p", [EmptyACK]), - ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(EmptyACK)). + ok = gen_udp:send(UdpSock, IpAddr, Port, emqx_coap_frame:serialize_pkt(EmptyACK, undefined)). test_send_coap_observe_ack(UdpSock, Host, Port, Code, Content, Request) -> is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), is_list(Host) orelse error("Host is not a string"), {ok, IpAddr} = inet:getaddr(Host, inet), - Response = lwm2m_coap_message:response(Code, Content, Request), - Response1 = lwm2m_coap_message:set(observe, 0, Response), + Response = response(Code, Content, Request), + Response1 = emqx_coap_message:set(observe, 0, Response), Response2 = Response1#coap_message{type = ack}, ?LOGT("test_send_coap_observe_ack Response=~p", [Response2]), - ResponseBinary = lwm2m_coap_message_parser:encode(Response2), + ResponseBinary = emqx_coap_frame:serialize_pkt(Response2, undefined), ok = gen_udp:send(UdpSock, IpAddr, Port, ResponseBinary). test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) -> @@ -1904,10 +1906,10 @@ test_send_coap_notif(UdpSock, Host, Port, Content, ObSeq, Request) -> is_list(Host) orelse error("Host is not a string"), {ok, IpAddr} = inet:getaddr(Host, inet), - Notif = lwm2m_coap_message:response({ok, content}, Content, Request), - NewNotif = lwm2m_coap_message:set(observe, ObSeq, Notif), + Notif = response({ok, content}, Content, Request), + NewNotif = emqx_coap_message:set(observe, ObSeq, Notif), ?LOGT("test_send_coap_notif Response=~p", [NewNotif]), - NotifBinary = lwm2m_coap_message_parser:encode(NewNotif), + NotifBinary = emqx_coap_frame:serialize_pkt(NewNotif, undefined), ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, NotifBinary]), ok = gen_udp:send(UdpSock, IpAddr, Port, NotifBinary). @@ -1952,30 +1954,18 @@ make_segment(Seg) -> get_coap_path(Options) -> - get_path(Options, <<>>). + Seps = maps:get(uri_path, Options, []), + lists:foldl(fun(Sep, Acc) -> + <> + end, + <<>>, + Seps). get_coap_query(Options) -> - proplists:get_value(uri_query, Options, []). + maps:get(uri_query, Options, #{}). get_coap_observe(Options) -> - get_observe(Options). - - -get_path([], Acc) -> - %?LOGT("get_path Acc=~p", [Acc]), - Acc; -get_path([{uri_path, Path1}|T], Acc) -> - %?LOGT("Path=~p, Acc=~p", [Path1, Acc]), - get_path(T, join_path(Path1, Acc)); -get_path([{_, _}|T], Acc) -> - get_path(T, Acc). - -get_observe([]) -> - undefined; -get_observe([{observe, V}|_T]) -> - V; -get_observe([{_, _}|T]) -> - get_observe(T). + maps:get(observe, Options, undefined). join_path([], Acc) -> Acc; join_path([<<"/">>|T], Acc) -> @@ -1985,3 +1975,10 @@ join_path([H|T], Acc) -> sprintf(Format, Args) -> lists:flatten(io_lib:format(Format, Args)). + +response(Code, #coap_content{content_format = Format, payload = Payload}, Req) -> + #coap_message{options = Opts} = Msg = emqx_coap_message:response(Code, Payload, Req), + Msg#coap_message{options = Opts#{content_format => Format}}. + +request(Type, Method, #coap_content{content_format = Format, payload = Payload}, Opts) -> + emqx_coap_message:request(Type, Method, Payload, [{content_format, Format} | Opts]). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl index c9d91748a..081f11005 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -24,7 +24,7 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). -include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). --include_lib("lwm2m_coap/include/coap.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -53,6 +53,14 @@ gateway.lwm2m { -define(assertExists(Map, Key), ?assertNotEqual(maps:get(Key, Map, undefined), undefined)). +-record(coap_content, {content_format, payload = <<>>}). + +-import(emqx_lwm2m_SUITE, [ request/4, response/3, test_send_coap_response/7 + , test_recv_coap_request/1, test_recv_coap_response/1 + , test_send_coap_request/6, test_recv_mqtt_response/1 + , std_register/5, reslove_uri/1, split_path/1, split_query/1 + , join_path/2, sprintf/2]). + %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- @@ -214,107 +222,3 @@ discover_received_request(ClientId, Path, Action) -> ?assertExists(Res, <<"path">>), ?assertExists(Res, <<"name">>), ?assertExists(Res, <<"operations">>). - -test_recv_mqtt_response(RespTopic) -> - receive - {publish, #{topic := RespTopic, payload := RM}} -> - ?LOGT("test_recv_mqtt_response Response=~p", [RM]), - RM - after 1000 -> timeout_test_recv_mqtt_response - end. - -test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Options) orelse error("Options must be a list"), - case resolve_uri(Uri) of - {coap, {IpAddr, Port}, Path, Query} -> - Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), - Request = Request0#coap_message{id = MsgId}, - ?LOGT("send_coap_request Request=~p", [Request]), - RequestBinary = lwm2m_coap_message_parser:encode(Request), - ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), - ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); - {SchemeDiff, ChIdDiff, _, _} -> - error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) - end. - -test_recv_coap_response(UdpSock) -> - {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), - Response = lwm2m_coap_message_parser:decode(Packet), - ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), - #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, - ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), - Response. - -test_recv_coap_request(UdpSock) -> - case gen_udp:recv(UdpSock, 0, 2000) of - {ok, {_Address, _Port, Packet}} -> - Request = lwm2m_coap_message_parser:decode(Packet), - #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, - ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), - Request; - {error, Reason} -> - ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]), - timeout_test_recv_coap_request - end. - -test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) -> - is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), - is_list(Host) orelse error("Host is not a string"), - - {ok, IpAddr} = inet:getaddr(Host, inet), - Response = lwm2m_coap_message:response(Code, Content, Request), - Response2 = case Ack of - true -> Response#coap_message{type = ack}; - false -> Response - end, - ?LOGT("test_send_coap_response Response=~p", [Response2]), - ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)). - -std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) -> - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = ObjectList}, - [], - MsgId1), - #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock), - test_recv_mqtt_response(RespTopic), - timer:sleep(100). - -resolve_uri(Uri) -> - {ok, #{scheme := Scheme, - host := Host, - port := PortNo, - path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri), - Query = maps:get(query, URIMap, ""), - {ok, PeerIP} = inet:getaddr(Host, inet), - {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}. - -split_path([]) -> []; -split_path([$/]) -> []; -split_path([$/ | Path]) -> split_segments(Path, $/, []). - -split_query([]) -> []; -split_query(Path) -> split_segments(Path, $&, []). - -split_segments(Path, Char, Acc) -> - case string:rchr(Path, Char) of - 0 -> - [make_segment(Path) | Acc]; - N when N > 0 -> - split_segments(string:substr(Path, 1, N-1), Char, - [make_segment(string:substr(Path, N+1)) | Acc]) - end. - -make_segment(Seg) -> - list_to_binary(emqx_http_lib:uri_decode(Seg)). - -join_path([], Acc) -> Acc; -join_path([<<"/">>|T], Acc) -> - join_path(T, Acc); -join_path([H|T], Acc) -> - join_path(T, <>). - -sprintf(Format, Args) -> - lists:flatten(io_lib:format(Format, Args)). From 63d3a7b52506d5462275be6fb8f51d41dbca019e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 13:38:09 +0800 Subject: [PATCH 18/84] feat(upload certs): save certs to file --- apps/emqx/src/emqx_authentication.erl | 106 +++++++++++++++++- apps/emqx/test/emqx_authentication_SUITE.erl | 53 ++++++++- apps/emqx_authn/src/emqx_authn_api.erl | 3 + .../src/emqx_connector_schema_lib.erl | 6 +- 4 files changed, 158 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 2e53d85eb..388f107e7 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -73,6 +73,11 @@ , code_change/3 ]). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + -define(CHAINS_TAB, emqx_authn_chains). -define(VER_1, <<"1">>). @@ -193,20 +198,31 @@ pre_config_update(UpdateReq, OldConfig) -> end. do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> - {ok, OldConfig ++ [Config]}; + try convert_cert_options(Config) of + NConfig -> + {ok, OldConfig ++ [NConfig]} + catch + error:{convert_cert_option, _} = Reason -> + {error, Reason} + end; do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> NewConfig = lists:filter(fun(OldConfig0) -> AuthenticatorID =/= generate_id(OldConfig0) end, OldConfig), {ok, NewConfig}; do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> - NewConfig = lists:map(fun(OldConfig0) -> + try lists:map(fun(OldConfig0) -> case AuthenticatorID =:= generate_id(OldConfig0) of - true -> maps:merge(OldConfig0, Config); + true -> convert_cert_options(Config, OldConfig0); false -> OldConfig0 end - end, OldConfig), - {ok, NewConfig}; + end, OldConfig) of + NewConfig -> + {ok, NewConfig} + catch + error:{convert_cert_option, _} = Reason -> + {error, Reason} + end; do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> case split_by_id(AuthenticatorID, OldConfig) of {error, Reason} -> {error, Reason}; @@ -600,6 +616,83 @@ reply(Reply, State) -> %% Internal functions %%------------------------------------------------------------------------------ +convert_cert_options(Config) -> + Keys = maps:keys(filter_empty(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], Config))), + lists:foldl(fun(Key, Acc) -> + convert_cert_option(Key, Acc) + end, Config, Keys). + +convert_cert_options(NewConfig, OldConfig) -> + Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], + NewCerts = maps:with(Keys, NewConfig), + OldCerts = maps:fold(fun(K, V, Acc) -> + {ok, Bin} = file:read_file(V), + Acc#{K => Bin} + end, #{}, maps:with(Keys, OldConfig)), + Diff = diff_certs(NewCerts, OldCerts), + lists:foldl(fun({identical, K}, Acc) -> + Acc#{K => maps:get(K, OldConfig)}; + ({T, K}, Acc) when T =:= added orelse T =:= changed -> + convert_cert_option(K, Acc) + end, NewConfig, Diff). + +convert_cert_option(Key, Config) -> + PemBin = maps:get(Key, Config), + case public_key:pem_decode(PemBin) =/= [] of + true -> + Filename = to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", generate_filename(Key)])), + case filelib:ensure_dir(Filename) of + ok -> + case file:write_file(Filename, PemBin) of + ok -> Config#{Key => Filename}; + {error, Reason} -> error({convert_cert_option, {write_file, Reason}}) + end; + {error, Reason} -> + error({convert_cert_option, {ensure_dir, Reason}}) + end; + false -> + error({convert_cert_option, invalid_certificate}) + end. + +generate_filename(Key) -> + Prefix = case Key of + <<"keyfile">> -> "key-"; + <<"certfile">> -> "cert-"; + <<"cacertfile">> -> "cacert-" + end, + Prefix ++ emqx_plugin_libs_id:gen() ++ ".pem". + +filter_empty(L) when is_list(L) -> + [I || I <- L, I =/= "" andalso I =/= undefined]; +filter_empty(M) when is_map(M) -> + maps:from_list(filter_empty(maps:to_list(M))). + +diff_certs(NewCerts0, OldCerts0) -> + NewCerts = filter_empty(NewCerts0), + OldCerts = filter_empty(OldCerts0), + Diff = lists:foldl(fun({OldK, OldPem}, Acc) -> + case maps:find(OldK, NewCerts) of + error -> + Acc; + {ok, NewPem} -> + case diff_cert(NewPem, OldPem) of + true -> + [{changed, OldK} | Acc]; + false -> + [{identical, OldK} | Acc] + end + end + end, + [], maps:to_list(OldCerts)), + Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(OldCerts), NewCerts))], + Diff ++ Added. + +diff_cert(Pem1, Pem2) -> + cal_md5_for_cert(Pem1) =/= cal_md5_for_cert(Pem2). + +cal_md5_for_cert(Pem) -> + crypto:hash(md5, term_to_binary(public_key:pem_decode(Pem))). + split_by_id(ID, AuthenticatorsConfig) -> case lists:foldl( fun(C, {P1, P2, F0}) -> @@ -777,3 +870,6 @@ to_list(M) when is_map(M) -> [M]; to_list(L) when is_list(L) -> L. + +to_bin(B) when is_binary(B) -> B; +to_bin(L) when is_list(L) -> list_to_binary(L). diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 001a4b40e..4af54f842 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -92,6 +92,19 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([]), ok. +init_per_testcase(_, Config) -> + meck:new(emqx, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx, get_config, fun([node, data_dir]) -> + {data_dir, Data} = lists:keyfind(data_dir, 1, Config), + Data; + (C) -> meck:passthrough([C]) + end), + Config. + +end_per_testcase(_, _Config) -> + meck:unload(emqx), + ok. + t_chain(_) -> % CRUD of authentication chain ChainName = 'test', @@ -203,7 +216,7 @@ t_update_config(_) -> ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID2)), - ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, #{}})), + ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, AuthenticatorConfig1#{enable => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch({ok, _}, update_config([authentication], {move_authenticator, Global, ID2, top})), @@ -220,7 +233,7 @@ t_update_config(_) -> ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID2)), - ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, #{}})), + ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, AuthenticatorConfig1#{enable => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), ?assertMatch({ok, _}, update_config(ConfKeyPath, {move_authenticator, ListenerID, ID2, top})), @@ -234,5 +247,41 @@ t_update_config(_) -> ?AUTHN:remove_provider(AuthNType2), ok. +t_convert_cert_options(_) -> + Certs = certs([ {<<"keyfile">>, "key.pem"} + , {<<"certfile">>, "cert.pem"} + , {<<"cacertfile">>, "cacert.pem"} + ]), + NCerts = ?AUTHN:convert_cert_options(Certs), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, Certs))), + + Certs2 = certs([ {<<"keyfile">>, "key.pem"} + , {<<"certfile">>, "cert.pem"} + ]), + NCerts2 = ?AUTHN:convert_cert_options(Certs2, NCerts), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, Certs2))), + ?assertEqual(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, NCerts2)), + ?assertEqual(maps:get(<<"certfile">>, NCerts), maps:get(<<"certfile">>, NCerts2)), + + Certs3 = certs([ {<<"keyfile">>, "client-key.pem"} + , {<<"certfile">>, "client-cert.pem"} + , {<<"cacertfile">>, "cacert.pem"} + ]), + NCerts3 = ?AUTHN:convert_cert_options(Certs3, NCerts2), + ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts3), maps:get(<<"keyfile">>, Certs3))), + ?assertNotEqual(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, NCerts3)), + ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)). + update_config(Path, ConfigRequest) -> emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + +certs(Certs) -> + CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"), + lists:foldl(fun({Key, Filename}, Acc) -> + {ok, Bin} = file:read_file(filename:join([CertsPath, Filename])), + Acc#{Key => Bin} + end, #{}, Certs). + +diff_cert(CertFile, CertPem2) -> + {ok, CertPem1} = file:read_file(CertFile), + ?AUTHN:diff_cert(CertPem1, CertPem2). \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index e492100ee..59897b51a 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1835,8 +1835,11 @@ find_listener(ListenerID) -> {ok, {Type, Name}} end. +% convert_tls_options(Config)-> + create_authenticator(ConfKeyPath, ChainName0, Config) -> ChainName = to_atom(ChainName0), + % {NConfig, Certs} = convert_tls_options(Config), case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 6dcc564af..ecdfb1416 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -107,15 +107,15 @@ auto_reconnect(default) -> true; auto_reconnect(_) -> undefined. cacertfile(type) -> string(); -cacertfile(default) -> ""; +cacertfile(nullable) -> true; cacertfile(_) -> undefined. keyfile(type) -> string(); -keyfile(default) -> ""; +keyfile(nullable) -> true; keyfile(_) -> undefined. certfile(type) -> string(); -certfile(default) -> ""; +certfile(nullable) -> true; certfile(_) -> undefined. verify(type) -> boolean(); From ee178ccea9ec70c222a183031164e7b3e78e10da Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 14:59:13 +0800 Subject: [PATCH 19/84] feat(upload certs): support return cert content by http api --- apps/emqx/src/emqx_authentication.erl | 94 ++++++++++---------- apps/emqx/test/emqx_authentication_SUITE.erl | 6 +- apps/emqx_authn/src/emqx_authn_api.erl | 24 +++-- 3 files changed, 68 insertions(+), 56 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 388f107e7..5b12ddb4f 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -198,7 +198,7 @@ pre_config_update(UpdateReq, OldConfig) -> end. do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> - try convert_cert_options(Config) of + try convert_certs(Config) of NConfig -> {ok, OldConfig ++ [NConfig]} catch @@ -213,7 +213,7 @@ do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldCon do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> try lists:map(fun(OldConfig0) -> case AuthenticatorID =:= generate_id(OldConfig0) of - true -> convert_cert_options(Config, OldConfig0); + true -> convert_certs(Config, OldConfig0); false -> OldConfig0 end end, OldConfig) of @@ -616,42 +616,48 @@ reply(Reply, State) -> %% Internal functions %%------------------------------------------------------------------------------ -convert_cert_options(Config) -> - Keys = maps:keys(filter_empty(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], Config))), - lists:foldl(fun(Key, Acc) -> - convert_cert_option(Key, Acc) - end, Config, Keys). +convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> + NSSLOPts = lists:foldl(fun(K, Acc) -> + case maps:get(K, Acc, undefined) of + undefined -> Acc; + PemBin -> + CertFile = generate_filename(K), + ok = save_cert_to_file(CertFile, PemBin), + Acc#{K => CertFile} + end + end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), + Config#{<<"ssl">> => NSSLOPts}; +convert_certs(Config) -> + Config. -convert_cert_options(NewConfig, OldConfig) -> - Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], - NewCerts = maps:with(Keys, NewConfig), - OldCerts = maps:fold(fun(K, V, Acc) -> - {ok, Bin} = file:read_file(V), - Acc#{K => Bin} - end, #{}, maps:with(Keys, OldConfig)), - Diff = diff_certs(NewCerts, OldCerts), - lists:foldl(fun({identical, K}, Acc) -> - Acc#{K => maps:get(K, OldConfig)}; - ({T, K}, Acc) when T =:= added orelse T =:= changed -> - convert_cert_option(K, Acc) - end, NewConfig, Diff). +convert_certs(#{<<"ssl">> := NewSSLOpts} = NewConfig, OldConfig) -> + OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), + Diff = diff_certs(NewSSLOpts, OldSSLOpts), + NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> + Acc#{K => maps:get(K, OldSSLOpts)}; + ({_, K}, Acc) -> + CertFile = generate_filename(K), + ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), + Acc#{K => CertFile} + end, NewSSLOpts, Diff), + NewConfig#{<<"ssl">> => NSSLOpts}; +convert_certs(NewConfig, _OldConfig) -> + NewConfig. -convert_cert_option(Key, Config) -> - PemBin = maps:get(Key, Config), +save_cert_to_file(Filename, PemBin) -> case public_key:pem_decode(PemBin) =/= [] of true -> - Filename = to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", generate_filename(Key)])), case filelib:ensure_dir(Filename) of ok -> case file:write_file(Filename, PemBin) of - ok -> Config#{Key => Filename}; - {error, Reason} -> error({convert_cert_option, {write_file, Reason}}) + ok -> ok; + {error, Reason} -> error({save_cert_to_file, {write_file, Reason}}) end; {error, Reason} -> - error({convert_cert_option, {ensure_dir, Reason}}) + error({save_cert_to_file, {ensure_dir, Reason}}) end; false -> - error({convert_cert_option, invalid_certificate}) + error({save_cert_to_file, invalid_certificate}) end. generate_filename(Key) -> @@ -660,31 +666,27 @@ generate_filename(Key) -> <<"certfile">> -> "cert-"; <<"cacertfile">> -> "cacert-" end, - Prefix ++ emqx_plugin_libs_id:gen() ++ ".pem". + to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_plugin_libs_id:gen() ++ ".pem"])). -filter_empty(L) when is_list(L) -> - [I || I <- L, I =/= "" andalso I =/= undefined]; -filter_empty(M) when is_map(M) -> - maps:from_list(filter_empty(maps:to_list(M))). - -diff_certs(NewCerts0, OldCerts0) -> - NewCerts = filter_empty(NewCerts0), - OldCerts = filter_empty(OldCerts0), - Diff = lists:foldl(fun({OldK, OldPem}, Acc) -> - case maps:find(OldK, NewCerts) of - error -> - Acc; - {ok, NewPem} -> - case diff_cert(NewPem, OldPem) of +diff_certs(NewSSLOpts, OldSSLOpts) -> + Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], + CertPems = maps:with(Keys, NewSSLOpts), + CertFiles = maps:with(Keys, OldSSLOpts), + Diff = lists:foldl(fun({K, CertFile}, Acc) -> + case maps:find(K, CertPems) of + error -> Acc; + {ok, PemBin1} -> + {ok, PemBin2} = file:read_file(CertFile), + case diff_cert(PemBin1, PemBin2) of true -> - [{changed, OldK} | Acc]; + [{changed, K} | Acc]; false -> - [{identical, OldK} | Acc] + [{identical, K} | Acc] end end end, - [], maps:to_list(OldCerts)), - Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(OldCerts), NewCerts))], + [], maps:to_list(CertFiles)), + Added = [{added, K} || K <- maps:keys(maps:without(maps:keys(CertFiles), CertPems))], Diff ++ Added. diff_cert(Pem1, Pem2) -> diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 4af54f842..ff219e64c 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -252,13 +252,13 @@ t_convert_cert_options(_) -> , {<<"certfile">>, "cert.pem"} , {<<"cacertfile">>, "cacert.pem"} ]), - NCerts = ?AUTHN:convert_cert_options(Certs), + #{<<"ssl">> := NCerts} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, Certs))), Certs2 = certs([ {<<"keyfile">>, "key.pem"} , {<<"certfile">>, "cert.pem"} ]), - NCerts2 = ?AUTHN:convert_cert_options(Certs2, NCerts), + #{<<"ssl">> := NCerts2} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs2}, #{<<"ssl">> => NCerts}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, Certs2))), ?assertEqual(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, NCerts2)), ?assertEqual(maps:get(<<"certfile">>, NCerts), maps:get(<<"certfile">>, NCerts2)), @@ -267,7 +267,7 @@ t_convert_cert_options(_) -> , {<<"certfile">>, "client-cert.pem"} , {<<"cacertfile">>, "cacert.pem"} ]), - NCerts3 = ?AUTHN:convert_cert_options(Certs3, NCerts2), + #{<<"ssl">> := NCerts3} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs3}, #{<<"ssl">> => NCerts2}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts3), maps:get(<<"keyfile">>, Certs3))), ?assertNotEqual(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, NCerts3)), ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 59897b51a..16f580d6a 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1835,23 +1835,20 @@ find_listener(ListenerID) -> {ok, {Type, Name}} end. -% convert_tls_options(Config)-> - create_authenticator(ConfKeyPath, ChainName0, Config) -> ChainName = to_atom(ChainName0), - % {NConfig, Certs} = convert_tls_options(Config), case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), - {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; {error, {_, _, Reason}} -> serialize_error(Reason) end. list_authenticators(ConfKeyPath) -> AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), - NAuthenticators = [maps:put(id, ?AUTHN:generate_id(AuthenticatorConfig), AuthenticatorConfig) + NAuthenticators = [maps:put(id, ?AUTHN:generate_id(AuthenticatorConfig), convert_certs(AuthenticatorConfig)) || AuthenticatorConfig <- AuthenticatorsConfig], {200, NAuthenticators}. @@ -1859,7 +1856,7 @@ list_authenticator(ConfKeyPath, AuthenticatorID) -> AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), case find_config(AuthenticatorID, AuthenticatorsConfig) of {ok, AuthenticatorConfig} -> - {200, AuthenticatorConfig#{id => AuthenticatorID}}; + {200, maps:put(id, AuthenticatorID, convert_certs(AuthenticatorConfig))}; {error, Reason} -> serialize_error(Reason) end. @@ -1870,7 +1867,7 @@ update_authenticator(ConfKeyPath, ChainName0, AuthenticatorID, Config) -> {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), - {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; {error, {_, _, Reason}} -> serialize_error(Reason) end. @@ -1974,6 +1971,19 @@ fill_defaults(Config) -> ?AUTHN, #{<<"authentication">> => Config}, #{nullable => true, no_conversion => true}), CheckedConfig. +convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> + NSSLOpts = lists:foldl(fun(K, Acc) -> + case maps:get(K, Acc, undefined) of + undefined -> Acc; + Filename -> + {ok, Bin} = file:read_file(Filename), + Acc#{K => Bin} + end + end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), + Config#{<<"ssl">> => NSSLOpts}; +convert_certs(Config) -> + Config. + serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => list_to_binary( From 1a61640b15f995b3cdc5369d4e5a436d5739d22a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 15:07:57 +0800 Subject: [PATCH 20/84] feat(upload certs): serialize errors about saving cert --- apps/emqx/src/emqx_authentication.erl | 4 ++-- apps/emqx_authn/src/emqx_authn_api.erl | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 5b12ddb4f..c35a7ea1b 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -202,7 +202,7 @@ do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> NConfig -> {ok, OldConfig ++ [NConfig]} catch - error:{convert_cert_option, _} = Reason -> + error:{save_cert_to_file, _} = Reason -> {error, Reason} end; do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> @@ -220,7 +220,7 @@ do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config} NewConfig -> {ok, NewConfig} catch - error:{convert_cert_option, _} = Reason -> + error:{save_cert_to_file, _} = Reason -> {error, Reason} end; do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 16f580d6a..93e8c4746 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -2024,6 +2024,16 @@ serialize_error(unsupported_operation) -> {400, #{code => <<"BAD_REQUEST">>, message => <<"Operation not supported in this authentication type">>}}; +serialize_error({save_cert_to_file, invalid_certificate}) -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"Invalid certificate">>}}; + +serialize_error({save_cert_to_file, {_, Reason}}) -> + {500, #{code => <<"INTERNAL_SERVER_ERROR">>, + message => list_to_binary( + io_lib:format("Cannot save certificate to file due to '~p'", [Reason]) + )}}; + serialize_error({missing_parameter, Name}) -> {400, #{code => <<"MISSING_PARAMETER">>, message => list_to_binary( From f6d7739f01ec1271f11741ad6e5dd4cb210e8e83 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 16:09:36 +0800 Subject: [PATCH 21/84] fix(upload certs): fix external dependency --- apps/emqx/src/emqx_authentication.erl | 2 +- apps/emqx/src/emqx_misc.erl | 37 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index c35a7ea1b..d4ab4cdf3 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -666,7 +666,7 @@ generate_filename(Key) -> <<"certfile">> -> "cert-"; <<"cacertfile">> -> "cacert-" end, - to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_plugin_libs_id:gen() ++ ".pem"])). + to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_misc:gen() ++ ".pem"])). diff_certs(NewSSLOpts, OldSSLOpts) -> Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index d45b6f7ce..ce98a3066 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -45,6 +45,8 @@ , index_of/2 , maybe_parse_ip/1 , ipv6_probe/1 + , gen/0 + , gen/1 ]). -export([ bin2hexstr_A_F/1 @@ -52,6 +54,8 @@ , hexstr2bin/1 ]). +-define(SHORT, 8). + %% @doc Parse v4 or v6 string format address to tuple. %% `Host' itself is returned if it's not an ip string. maybe_parse_ip(Host) -> @@ -298,6 +302,39 @@ hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0; hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10; hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10. +-spec(gen() -> list()). +gen() -> + gen(?SHORT). + +-spec(gen(integer()) -> list()). +gen(Len) -> + BitLen = Len * 4, + <> = crypto:strong_rand_bytes(Len div 2), + int_to_hex(R, Len). + +%%------------------------------------------------------------------------------ +%% Internal Functions +%%------------------------------------------------------------------------------ + +int_to_hex(I, N) when is_integer(I), I >= 0 -> + int_to_hex([], I, 1, N). + +int_to_hex(L, I, Count, N) + when I < 16 -> + pad([int_to_hex(I) | L], N - Count); +int_to_hex(L, I, Count, N) -> + int_to_hex([int_to_hex(I rem 16) | L], I div 16, Count + 1, N). + +int_to_hex(I) when 0 =< I, I =< 9 -> + I + $0; +int_to_hex(I) when 10 =< I, I =< 15 -> + (I - 10) + $a. + +pad(L, 0) -> + L; +pad(L, Count) -> + pad([$0 | L], Count - 1). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). From ce6c24a8666b8e34bd353d8417428ee83d87303a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 14 Sep 2021 17:42:07 +0800 Subject: [PATCH 22/84] fix(authn): fix spelling error --- apps/emqx_authn/src/emqx_authn_app.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 016decdd2..4d4a22791 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -60,7 +60,7 @@ initialize() -> providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} - , {{'password-based', posgresql}, emqx_authn_pgsql} + , {{'password-based', postgresql}, emqx_authn_pgsql} , {{'password-based', mongodb}, emqx_authn_mongodb} , {{'password-based', redis}, emqx_authn_redis} , {{'password-based', 'http-server'}, emqx_authn_http} From 0fe300575e7c521deabaf1ab3133aeb1ce40c898 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 15 Sep 2021 09:59:35 +0800 Subject: [PATCH 23/84] chore(authn): better function name --- apps/emqx/src/emqx_authentication.erl | 2 +- apps/emqx/src/emqx_misc.erl | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index d4ab4cdf3..483305aa4 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -666,7 +666,7 @@ generate_filename(Key) -> <<"certfile">> -> "cert-"; <<"cacertfile">> -> "cacert-" end, - to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_misc:gen() ++ ".pem"])). + to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_misc:gen_id() ++ ".pem"])). diff_certs(NewSSLOpts, OldSSLOpts) -> Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index ce98a3066..446039778 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -45,8 +45,8 @@ , index_of/2 , maybe_parse_ip/1 , ipv6_probe/1 - , gen/0 - , gen/1 + , gen_id/0 + , gen_id/1 ]). -export([ bin2hexstr_A_F/1 @@ -302,12 +302,12 @@ hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0; hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10; hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10. --spec(gen() -> list()). -gen() -> - gen(?SHORT). +-spec(gen_id() -> list()). +gen_id() -> + gen_id(?SHORT). --spec(gen(integer()) -> list()). -gen(Len) -> +-spec(gen_id(integer()) -> list()). +gen_id(Len) -> BitLen = Len * 4, <> = crypto:strong_rand_bytes(Len div 2), int_to_hex(R, Len). From 237d3c0543819b9b13895261919707b7d070ead4 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:40:05 +0800 Subject: [PATCH 24/84] Auto subscribe format (#5740) * fix: auto sub list format with default --- .../etc/emqx_auto_subscribe.conf | 46 +++++++++---------- .../src/emqx_auto_subscribe.erl | 13 +++++- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf index f6d041dab..becc87516 100644 --- a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf +++ b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf @@ -1,28 +1,28 @@ auto_subscribe { topics = [ - # { - # topic = "/c/${clientid}", - # qos = 0 - # rh = 0 - # rap = 0 - # nl = 0 - # } - # { - # topic = "/u/${username}", - # }, - # { - # topic = "/h/${host}", - # qos = 2 - # }, - # { - # topic = "/p/${port}", - # }, - # { - # topic = "/topic/abc", - # }, - # { - # topic = "/client/${clientid}/username/${username}/host/${host}/port/${port}", - # } + ## { + ## topic = "/c/${clientid}" + ## qos = 0 + ## rh = 0 + ## rap = 0 + ## nl = 0 + ## }, + ## { + ## topic = "/u/${username}" + ## }, + ## { + ## topic = "/h/${host}" + ## qos = 2 + ## }, + ## { + ## topic = "/p/${port}" + ## }, + ## { + ## topic = "/topic/abc" + ## }, + ## { + ## topic = "/client/${clientid}/username/${username}/host/${host}/port/${port}" + ## } ] } diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl index b13b3760a..56bbf3a1c 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl @@ -38,7 +38,7 @@ max_limit() -> ?MAX_AUTO_SUBSCRIBE. list() -> - emqx:get_config([auto_subscribe, topics], []). + format(emqx:get_config([auto_subscribe, topics], [])). update(Topics) -> update_(Topics). @@ -68,6 +68,17 @@ on_client_connected(_, _, _) -> %%-------------------------------------------------------------------- %% internal +format(Rules) when is_list(Rules) -> + [format(Rule) || Rule <- Rules]; +format(Rule = #{topic := Topic}) when is_map(Rule) -> + #{ + topic => Topic, + qos => maps:get(qos, Rule, 0), + rh => maps:get(rh, Rule, 0), + rap => maps:get(rap, Rule, 0), + nl => maps:get(nl, Rule, 0) + }. + update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE -> {ok, _} = emqx:update_config([auto_subscribe, topics], Topics), update_hook(); From 6c27c83dc3f2ab52151419a962ac590eecf0892e Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 15 Sep 2021 11:12:16 +0800 Subject: [PATCH 25/84] fix(emqx_schema): support get ip address by host name --- apps/emqx/src/emqx_schema.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ed3f64d0e..7ff517c31 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1093,9 +1093,18 @@ to_bar_separated_list(Str) -> to_ip_port(Str) -> case string:tokens(Str, ":") of [Ip, Port] -> + PortVal = list_to_integer(Port), case inet:parse_address(Ip) of - {ok, R} -> {ok, {R, list_to_integer(Port)}}; - _ -> {error, Str} + {ok, R} -> + {ok, {R, PortVal}}; + _ -> + %% check is a rfc1035's hostname + case inet_parse:domain(Ip) of + true -> + {ok, {Ip, PortVal}}; + _ -> + {error, Str} + end end; _ -> {error, Str} end. From 304c5613acf70430bff566ebe6c98978d9c10ae2 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 14 Sep 2021 11:36:20 +0800 Subject: [PATCH 26/84] refactor(resource): rename the emqx_resource:update/4 to recreate/4 --- apps/emqx_authz/src/emqx_authz.erl | 2 +- apps/emqx_resource/src/emqx_resource.erl | 47 ++++++++++--------- .../src/emqx_resource_instance.erl | 8 ++-- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 9ba67ddd1..6c7ceb6c2 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -392,7 +392,7 @@ gen_id(Type) -> create_resource(#{type := DB, annotations := #{id := ResourceID}} = Source) -> - case emqx_resource:update(ResourceID, connector_module(DB), Source, []) of + case emqx_resource:recreate(ResourceID, connector_module(DB), Source, []) of {ok, _} -> ResourceID; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index cad32bcb2..02b49c47f 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -40,8 +40,8 @@ -export([ check_config/2 , check_and_create/3 , check_and_create_local/3 - , check_and_update/4 - , check_and_update_local/4 + , check_and_recreate/4 + , check_and_recreate_local/4 , resource_type_from_str/1 ]). @@ -50,10 +50,10 @@ %% todo: replicate operations -export([ create/3 %% store the config and start the instance , create_local/3 - , create_dry_run/3 %% run start/2, health_check/2 and stop/1 sequentially - , create_dry_run_local/3 - , update/4 %% update the config, stop the old instance and start the new one - , update_local/4 + , create_dry_run/2 %% run start/2, health_check/2 and stop/1 sequentially + , create_dry_run_local/2 + , recreate/4 %% this will do create_dry_run, stop the old instance and start a new one + , recreate_local/4 , remove/1 %% remove the config and stop the instance , remove_local/1 ]). @@ -164,25 +164,26 @@ create(InstId, ResourceType, Config) -> create_local(InstId, ResourceType, Config) -> call_instance(InstId, {create, InstId, ResourceType, Config}). --spec create_dry_run(instance_id(), resource_type(), resource_config()) -> +-spec create_dry_run(resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run(InstId, ResourceType, Config) -> - cluster_call(create_dry_run_local, [InstId, ResourceType, Config]). +create_dry_run(ResourceType, Config) -> + cluster_call(create_dry_run_local, [ResourceType, Config]). --spec create_dry_run_local(instance_id(), resource_type(), resource_config()) -> +-spec create_dry_run_local(resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run_local(InstId, ResourceType, Config) -> +create_dry_run_local(ResourceType, Config) -> + InstId = emqx_plugin_libs_id:gen(16), call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). --spec update(instance_id(), resource_type(), resource_config(), term()) -> +-spec recreate(instance_id(), resource_type(), resource_config(), term()) -> {ok, resource_data()} | {error, Reason :: term()}. -update(InstId, ResourceType, Config, Params) -> - cluster_call(update_local, [InstId, ResourceType, Config, Params]). +recreate(InstId, ResourceType, Config, Params) -> + cluster_call(recreate_local, [InstId, ResourceType, Config, Params]). --spec update_local(instance_id(), resource_type(), resource_config(), term()) -> +-spec recreate_local(instance_id(), resource_type(), resource_config(), term()) -> {ok, resource_data()} | {error, Reason :: term()}. -update_local(InstId, ResourceType, Config, Params) -> - call_instance(InstId, {update, InstId, ResourceType, Config, Params}). +recreate_local(InstId, ResourceType, Config, Params) -> + call_instance(InstId, {recreate, InstId, ResourceType, Config, Params}). -spec remove(instance_id()) -> ok | {error, Reason :: term()}. remove(InstId) -> @@ -296,17 +297,17 @@ check_and_create_local(InstId, ResourceType, RawConfig) -> check_and_do(ResourceType, RawConfig, fun(InstConf) -> create_local(InstId, ResourceType, InstConf) end). --spec check_and_update(instance_id(), resource_type(), raw_resource_config(), term()) -> +-spec check_and_recreate(instance_id(), resource_type(), raw_resource_config(), term()) -> {ok, resource_data()} | {error, term()}. -check_and_update(InstId, ResourceType, RawConfig, Params) -> +check_and_recreate(InstId, ResourceType, RawConfig, Params) -> check_and_do(ResourceType, RawConfig, - fun(InstConf) -> update(InstId, ResourceType, InstConf, Params) end). + fun(InstConf) -> recreate(InstId, ResourceType, InstConf, Params) end). --spec check_and_update_local(instance_id(), resource_type(), raw_resource_config(), term()) -> +-spec check_and_recreate_local(instance_id(), resource_type(), raw_resource_config(), term()) -> {ok, resource_data()} | {error, term()}. -check_and_update_local(InstId, ResourceType, RawConfig, Params) -> +check_and_recreate_local(InstId, ResourceType, RawConfig, Params) -> check_and_do(ResourceType, RawConfig, - fun(InstConf) -> update_local(InstId, ResourceType, InstConf, Params) end). + fun(InstConf) -> recreate_local(InstId, ResourceType, InstConf, Params) end). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> case check_config(ResourceType, RawConfig) of diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index 84b5a1f7c..f3587ff5a 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -107,8 +107,8 @@ handle_call({create, InstId, ResourceType, Config}, _From, State) -> handle_call({create_dry_run, InstId, ResourceType, Config}, _From, State) -> {reply, do_create_dry_run(InstId, ResourceType, Config), State}; -handle_call({update, InstId, ResourceType, Config, Params}, _From, State) -> - {reply, do_update(InstId, ResourceType, Config, Params), State}; +handle_call({recreate, InstId, ResourceType, Config, Params}, _From, State) -> + {reply, do_recreate(InstId, ResourceType, Config, Params), State}; handle_call({remove, InstId}, _From, State) -> {reply, do_remove(InstId), State}; @@ -141,8 +141,8 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ %% suppress the race condition check, as these functions are protected in gproc workers --dialyzer({nowarn_function, [do_update/4, do_create/3, do_restart/1, do_stop/1, do_health_check/1]}). -do_update(InstId, ResourceType, NewConfig, Params) -> +-dialyzer({nowarn_function, [do_recreate/4, do_create/3, do_restart/1, do_stop/1, do_health_check/1]}). +do_recreate(InstId, ResourceType, NewConfig, Params) -> case lookup(InstId) of {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> Config = emqx_resource:call_config_merge(ResourceType, OldConfig, From cb8dabe579d9f6b888cced4c48ee345aa63e5630 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Tue, 14 Sep 2021 17:32:26 +0800 Subject: [PATCH 27/84] feat(bridges): add CRUD HTTP APIs for bridges --- apps/emqx/rebar.config | 4 +- apps/emqx_bridge/etc/emqx_bridge.conf | 90 +++--- apps/emqx_bridge/src/emqx_bridge.erl | 170 +++++++++-- apps/emqx_bridge/src/emqx_bridge_api.erl | 298 +++++++++++-------- apps/emqx_bridge/src/emqx_bridge_app.erl | 20 +- apps/emqx_bridge/src/emqx_bridge_monitor.erl | 2 +- rebar.config | 6 +- 7 files changed, 382 insertions(+), 208 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a8462ad82..119d521fc 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -11,11 +11,11 @@ {deps, [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.1"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index bfba34e7c..c04abf82a 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -2,48 +2,48 @@ ## EMQ X Bridge ##-------------------------------------------------------------------- -#bridges.mqtt.my_mqtt_bridge_to_aws { -# server = "127.0.0.1:1883" -# proto_ver = "v4" -# username = "username1" -# password = "" -# clean_start = true -# keepalive = 300 -# retry_interval = "30s" -# max_inflight = 32 -# reconnect_interval = "30s" -# bridge_mode = true -# replayq { -# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" -# seg_bytes = "100MB" -# offload = false -# max_total_bytes = "1GB" -# } -# ssl { -# enable = false -# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" -# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" -# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" -# } -# ## we will create one MQTT connection for each element of the `message_in` -# message_in: [{ -# ## the `id` will be used as part of the clientid -# id = "pull_msgs_from_aws" -# subscribe_remote_topic = "aws/#" -# subscribe_qos = 1 -# local_topic = "from_aws/${topic}" -# payload = "${payload}" -# qos = "${qos}" -# retain = "${retain}" -# }] -# ## we will create one MQTT connection for each element of the `message_out` -# message_out: [{ -# ## the `id` will be used as part of the clientid -# id = "push_msgs_to_aws" -# subscribe_local_topic = "emqx/#" -# remote_topic = "from_emqx/${topic}" -# payload = "${payload}" -# qos = 1 -# retain = false -# }] -#} +bridges.mqtt.my_mqtt_bridge_to_aws { + server = "127.0.0.1:1883" + proto_ver = "v4" + username = "username1" + password = "" + clean_start = true + keepalive = 300 + retry_interval = "30s" + max_inflight = 32 + reconnect_interval = "30s" + bridge_mode = true + replayq { + dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" + seg_bytes = "100MB" + offload = false + max_total_bytes = "1GB" + } + ssl { + enable = false + keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" + certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" + cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + } + ## we will create one MQTT connection for each element of the `message_in` + message_in: [{ + ## the `id` will be used as part of the clientid + id = "pull_msgs_from_aws" + subscribe_remote_topic = "aws/#" + subscribe_qos = 1 + local_topic = "from_aws/${topic}" + payload = "${payload}" + qos = "${qos}" + retain = "${retain}" + }] + ## we will create one MQTT connection for each element of the `message_out` + message_out: [{ + ## the `id` will be used as part of the clientid + id = "push_msgs_to_aws" + subscribe_local_topic = "emqx/#" + remote_topic = "from_emqx/${topic}" + payload = "${payload}" + qos = 1 + retain = false + }] +} diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 75ebfac0c..4d2b80ba7 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -14,21 +14,34 @@ %% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_bridge). +-behaviour(emqx_config_handler). + +-export([post_config_update/4]). -export([ load_bridges/0 - , resource_type/1 - , bridge_type/1 - , name_to_resource_id/1 - , resource_id_to_name/1 + , get_bridge/2 + , get_bridge/3 , list_bridges/0 - , is_bridge/1 - , config_key_path/0 - , update_config/1 + , create_bridge/3 + , remove_bridge/3 + , update_bridge/3 + , start_bridge/2 + , stop_bridge/2 + , restart_bridge/2 ]). -load_bridges() -> - Bridges = emqx:get_config([bridges], #{}), - emqx_bridge_monitor:ensure_all_started(Bridges). +-export([ config_key_path/0 + ]). + +-export([ resource_type/1 + , bridge_type/1 + , resource_id/1 + , resource_id/2 + , parse_bridge_id/1 + ]). + +config_key_path() -> + [bridges]. resource_type(mqtt) -> emqx_connector_mqtt; resource_type(mysql) -> emqx_connector_mysql; @@ -44,27 +57,136 @@ bridge_type(emqx_connector_mongo) -> mongo; bridge_type(emqx_connector_redis) -> redis; bridge_type(emqx_connector_ldap) -> ldap. -name_to_resource_id(BridgeName) -> - Name = bin(BridgeName), - <<"bridge:", Name/binary>>. +post_config_update(_Req, NewConf, OldConf, _AppEnv) -> + #{added := Added, removed := Removed, changed := Updated} + = diff_confs(NewConf, OldConf), + perform_bridge_changes([ + {fun remove_bridge/3, Removed}, + {fun create_bridge/3, Added}, + {fun update_bridge/3, Updated} + ]). -resource_id_to_name(<<"bridge:", BridgeName/binary>> = _ResourceId) -> - BridgeName. +perform_bridge_changes(Tasks) -> + perform_bridge_changes(Tasks, ok). + +perform_bridge_changes([], Result) -> + Result; +perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> + Result = maps:fold(fun + ({_Type, _Name}, _Conf, {error, Reason}) -> + {error, Reason}; + ({Type, Name}, Conf, _) -> + case Action(Type, Name, Conf) of + {error, Reason} -> {error, Reason}; + Return -> Return + end + end, Result0, MapConfs), + perform_bridge_changes(Tasks, Result). + +load_bridges() -> + Bridges = emqx:get_config([bridges], #{}), + emqx_bridge_monitor:ensure_all_started(Bridges). + +resource_id(BridgeId) when is_binary(BridgeId) -> + <<"bridge:", BridgeId/binary>>. + +resource_id(BridgeType, BridgeName) -> + BridgeId = bridge_id(BridgeType, BridgeName), + resource_id(BridgeId). + +bridge_id(BridgeType, BridgeName) -> + Name = bin(BridgeName), + Type = bin(BridgeType), + <>. + +parse_bridge_id(BridgeId) -> + try + [Type, Name] = string:split(str(BridgeId), ":", leading), + {list_to_existing_atom(Type), list_to_atom(Name)} + catch + _ : _ -> error({invalid_bridge_id, BridgeId}) + end. list_bridges() -> - emqx_resource_api:list_instances(fun emqx_bridge:is_bridge/1). + lists:foldl(fun({Type, NameAndConf}, Bridges) -> + lists:foldl(fun({Name, RawConf}, Acc) -> + case get_bridge(Type, Name, RawConf) of + {error, not_found} -> Acc; + {ok, Res} -> [Res | Acc] + end + end, Bridges, maps:to_list(NameAndConf)) + end, [], maps:to_list(emqx:get_raw_config([bridges]))). -is_bridge(#{id := <<"bridge:", _/binary>>}) -> - true; -is_bridge(_Data) -> - false. +get_bridge(Type, Name) -> + RawConf = emqx:get_raw_config([bridges, Type, Name], #{}), + get_bridge(Type, Name, RawConf). +get_bridge(Type, Name, RawConf) -> + case emqx_resource:get_instance(resource_id(Type, Name)) of + {error, not_found} -> {error, not_found}; + {ok, Data} -> {ok, #{id => bridge_id(Type, Name), resource_data => Data, + raw_config => RawConf}} + end. -config_key_path() -> - [emqx_bridge, bridges]. +start_bridge(Type, Name) -> + restart_bridge(Type, Name). -update_config(ConfigReq) -> - emqx:update_config(config_key_path(), ConfigReq). +stop_bridge(Type, Name) -> + emqx_resource:stop(resource_id(Type, Name)). + +restart_bridge(Type, Name) -> + emqx_resource:restart(resource_id(Type, Name)). + +create_bridge(Type, Name, Conf) -> + ResId = resource_id(Type, Name), + case emqx_resource:create(ResId, + emqx_bridge:resource_type(Type), Conf) of + {ok, already_created} -> + emqx_resource:get_instance(ResId); + {ok, Data} -> + {ok, Data}; + {error, Reason} -> + {error, Reason} + end. + +update_bridge(Type, Name, Conf) -> + %% TODO: sometimes its not necessary to restart the bridge connection. + %% + %% - if the connection related configs like `username` is updated, we should restart/start + %% or stop bridges according to the change. + %% - if the connection related configs are not update, but channel configs `message_in` or + %% `message_out` are changed, then we should not restart the bridge, we only restart/start + %% the channels. + %% + emqx_resource:recreate(resource_id(Type, Name), + emqx_bridge:resource_type(Type), Conf). + +remove_bridge(Type, Name, _Conf) -> + case emqx_resource:remove(resource_id(Type, Name)) of + ok -> ok; + {error, not_found} -> ok; + {error, Reason} -> + {error, Reason} + end. + +diff_confs(NewConfs, OldConfs) -> + emqx_map_lib:diff_maps(flatten_confs(NewConfs), + flatten_confs(OldConfs)). + +flatten_confs(Conf0) -> + maps:from_list( + lists:append([do_flatten_confs(Type, Conf) + || {Type, Conf} <- maps:to_list(Conf0)])). + +do_flatten_confs(Type, Conf0) -> + [{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)]. bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c10875e55..56c421e0c 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -15,128 +15,194 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_api). --rest_api(#{ name => list_data_bridges - , method => 'GET' - , path => "/data_bridges" - , func => list_bridges - , descr => "List all data bridges" - }). +-behaviour(minirest_api). --rest_api(#{ name => get_data_bridge - , method => 'GET' - , path => "/data_bridges/:bin:name" - , func => get_bridge - , descr => "Get a data bridge by name" - }). - --rest_api(#{ name => create_data_bridge - , method => 'POST' - , path => "/data_bridges/:bin:name" - , func => create_bridge - , descr => "Create a new data bridge" - }). - --rest_api(#{ name => update_data_bridge - , method => 'PUT' - , path => "/data_bridges/:bin:name" - , func => update_bridge - , descr => "Update an existing data bridge" - }). - --rest_api(#{ name => delete_data_bridge - , method => 'DELETE' - , path => "/data_bridges/:bin:name" - , func => delete_bridge - , descr => "Delete an existing data bridge" - }). +-export([api_spec/0]). -export([ list_bridges/2 - , get_bridge/2 - , create_bridge/2 - , update_bridge/2 - , delete_bridge/2 + , list_local_bridges/1 + , crud_bridges_cluster/2 + , crud_bridges/3 ]). --define(BRIDGE(N, T, C), #{<<"name">> => N, <<"type">> => T, <<"config">> => C}). +-define(TYPES, [mqtt]). +-define(BRIDGE(N, T, C), #{<<"id">> => N, <<"type">> => T, <<"config">> => C}). +-define(TRY_PARSE_ID(ID, EXPR), + try emqx_bridge:parse_bridge_id(Id) of + {BridgeType, BridgeName} -> EXPR + catch + error:{invalid_bridge_id, Id0} -> + {400, #{code => 102, message => <<"invalid_bridge_id: ", Id0/binary>>}} + end). -list_bridges(_Binding, _Params) -> - {200, #{code => 0, data => [format_api_reply(Data) || - Data <- emqx_bridge:list_bridges()]}}. +req_schema() -> + Schema = [ + case maps:to_list(emqx:get_raw_config([bridges, T], #{})) of + %% the bridge is not configured, so we have no method to get the schema + [] -> #{}; + [{_K, Conf} | _] -> + emqx_mgmt_api_configs:gen_schema(Conf) + end + || T <- ?TYPES], + #{oneOf => Schema}. -get_bridge(#{name := Name}, _Params) -> - case emqx_resource:get_instance(emqx_bridge:name_to_resource_id(Name)) of - {ok, Data} -> - {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; +resp_schema() -> + #{oneOf := Schema} = req_schema(), + AddMetadata = fun(Prop) -> + Prop#{is_connected => #{type => boolean}, + id => #{type => string}, + bridge_type => #{type => string, enum => ?TYPES}, + node => #{type => string}} + end, + Schema1 = [S#{properties => AddMetadata(Prop)} + || S = #{properties := Prop} <- Schema], + #{oneOf => Schema1}. + +api_spec() -> + {bridge_apis(), []}. + +bridge_apis() -> + [list_all_bridges_api(), crud_bridges_apis(), operation_apis()]. + +list_all_bridges_api() -> + Metadata = #{ + get => #{ + description => <<"List all created bridges">>, + responses => #{ + <<"200">> => emqx_mgmt_util:array_schema(resp_schema(), + <<"A list of the bridges">>) + } + } + }, + {"/bridges/", Metadata, list_bridges}. + +crud_bridges_apis() -> + ReqSchema = req_schema(), + RespSchema = resp_schema(), + Metadata = #{ + get => #{ + description => <<"Get a bridge by Id">>, + parameters => [param_path_id()], + responses => #{ + <<"200">> => emqx_mgmt_util:array_schema(RespSchema, + <<"The details of the bridge">>), + <<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND']) + } + }, + put => #{ + description => <<"Create or update a bridge">>, + parameters => [param_path_id()], + 'requestBody' => emqx_mgmt_util:schema(ReqSchema), + responses => #{ + <<"200">> => emqx_mgmt_util:array_schema(RespSchema, <<"Bridge updated">>), + <<"400">> => emqx_mgmt_util:error_schema(<<"Update bridge failed">>, + ['UPDATE_FAILED']) + } + }, + delete => #{ + description => <<"Delete a bridge">>, + parameters => [param_path_id()], + responses => #{ + <<"200">> => emqx_mgmt_util:schema(<<"Bridge deleted">>), + <<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND']) + } + } + }, + {"/bridges/:id", Metadata, crud_bridges_cluster}. + +operation_apis() -> + Metadata = #{ + post => #{ + description => <<"Restart bridges on all nodes in the cluster">>, + parameters => [ + param_path_id(), + param_path_operation()], + responses => #{ + <<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), + <<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}}, + {"/bridges/:id/operation/:operation", Metadata, manage_bridges}. + +param_path_id() -> + #{ + name => id, + in => path, + schema => #{type => string}, + required => true + }. + +param_path_operation()-> + #{ + name => operation, + in => path, + required => true, + schema => #{ + type => string, + enum => [start, stop, restart]}, + example => restart + }. + +list_bridges(get, _Params) -> + {200, lists:append([list_local_bridges(Node) || Node <- ekka_mnesia:running_nodes()])}. + +list_local_bridges(Node) when Node =:= node() -> + [format_resp(Data) || Data <- emqx_bridge:list_bridges()]; +list_local_bridges(Node) -> + rpc_call(Node, list_local_bridges, [Node]). + +crud_bridges_cluster(Method, Params) -> + Results = [crud_bridges(Node, Method, Params) || Node <- ekka_mnesia:running_nodes()], + case lists:filter(fun({200}) -> false; ({200, _}) -> false; (_) -> true end, Results) of + [] -> + case Results of + [{200} | _] -> {200}; + _ -> {200, [Res || {200, Res} <- Results]} + end; + Errors -> + hd(Errors) + end. + +crud_bridges(Node, Method, Params) when Node =/= node() -> + rpc_call(Node, crud_bridges, [Node, Method, Params]); + +crud_bridges(_, get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, case emqx_bridge:get_bridge(BridgeType, BridgeName) of + {ok, Data} -> {200, format_resp(Data)}; {error, not_found} -> - {404, #{code => 102, message => <<"not_found: ", Name/binary>>}} + {404, #{code => 102, message => <<"not_found: ", Id/binary>>}} + end); + +crud_bridges(_, put, #{bindings := #{id := Id}, body := Conf}) -> + ?TRY_PARSE_ID(Id, + case emqx:update_config(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], Conf, + #{rawconf_with_defaults => true}) of + {ok, #{raw_config := RawConf, post_config_update := #{emqx_bridge := Data}}} -> + {200, format_resp(#{id => Id, raw_config => RawConf, resource_data => Data})}; + {ok, _} -> %% the bridge already exits + {ok, Data} = emqx_bridge:get_bridge(BridgeType, BridgeName), + {200, format_resp(Data)}; + {error, Reason} -> + {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} + end); + +crud_bridges(_, delete, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, + case emqx:remove_config(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName]) of + {ok, _} -> {200}; + {error, Reason} -> + {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} + end). + +format_resp(#{id := Id, raw_config := RawConf, resource_data := #{mod := Mod, status := Status}}) -> + IsConnected = fun(started) -> true; (_) -> false end, + RawConf#{ + id => Id, + node => node(), + bridge_type => emqx_bridge:bridge_type(Mod), + is_connected => IsConnected(Status) + }. + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res end. - -create_bridge(#{name := Name}, Params) -> - Config = proplists:get_value(<<"config">>, Params), - BridgeType = proplists:get_value(<<"type">>, Params), - case emqx_resource:check_and_create( - emqx_bridge:name_to_resource_id(Name), - emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config)) of - {ok, already_created} -> - {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; - {ok, Data} -> - update_config_and_reply(Name, BridgeType, Config, Data); - {error, Reason0} -> - Reason = emqx_resource_api:stringnify(Reason0), - {500, #{code => 102, message => <<"create bridge ", Name/binary, - " failed:", Reason/binary>>}} - end. - -update_bridge(#{name := Name}, Params) -> - Config = proplists:get_value(<<"config">>, Params), - BridgeType = proplists:get_value(<<"type">>, Params), - case emqx_resource:check_and_update( - emqx_bridge:name_to_resource_id(Name), - emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config), []) of - {ok, Data} -> - update_config_and_reply(Name, BridgeType, Config, Data); - {error, not_found} -> - {400, #{code => 102, message => <<"bridge not_found: ", Name/binary>>}}; - {error, Reason0} -> - Reason = emqx_resource_api:stringnify(Reason0), - {500, #{code => 102, message => <<"update bridge ", Name/binary, - " failed:", Reason/binary>>}} - end. - -delete_bridge(#{name := Name}, _Params) -> - case emqx_resource:remove(emqx_bridge:name_to_resource_id(Name)) of - ok -> delete_config_and_reply(Name); - {error, Reason} -> - {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} - end. - -format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> - #{type => emqx_bridge:bridge_type(Type), - name => emqx_bridge:resource_id_to_name(Id), - config => Conf, status => Status}. - -% format_conf(#{resource_type := Type, id := Id, config := Conf}) -> -% #{type => Type, name => emqx_bridge:resource_id_to_name(Id), -% config => Conf}. - -% get_all_configs() -> -% [format_conf(Data) || Data <- emqx_bridge:list_bridges()]. - -update_config_and_reply(Name, BridgeType, Config, Data) -> - case emqx_bridge:update_config({update, ?BRIDGE(Name, BridgeType, Config)}) of - {ok, _} -> - {200, #{code => 0, data => format_api_reply( - emqx_resource_api:format_data(Data))}}; - {error, Reason} -> - {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} - end. - -delete_config_and_reply(Name) -> - case emqx_bridge:update_config({delete, Name}) of - {ok, _} -> {200, #{code => 0, data => #{}}}; - {error, Reason} -> - {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} - end. - -atom(B) when is_binary(B) -> - list_to_existing_atom(binary_to_list(B)). diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index cfefe118f..004b32787 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -17,29 +17,15 @@ -behaviour(application). --behaviour(emqx_config_handler). - --export([start/2, stop/1, pre_config_update/2]). +-export([start/2, stop/1]). start(_StartType, _StartArgs) -> {ok, Sup} = emqx_bridge_sup:start_link(), ok = emqx_bridge:load_bridges(), - emqx_config_handler:add_handler(emqx_bridge:config_key_path(), ?MODULE), + emqx_config_handler:add_handler(emqx_bridge:config_key_path(), emqx_bridge), {ok, Sup}. stop(_State) -> ok. -%% internal functions -pre_config_update({update, Bridge = #{<<"name">> := Name}}, OldConf) -> - {ok, [Bridge | remove_bridge(Name, OldConf)]}; -pre_config_update({delete, Name}, OldConf) -> - {ok, remove_bridge(Name, OldConf)}; -pre_config_update(NewConf, _OldConf) when is_list(NewConf) -> - %% overwrite the entire config! - {ok, NewConf}. - -remove_bridge(_Name, undefined) -> - []; -remove_bridge(Name, OldConf) -> - [B || B = #{<<"name">> := Name0} <- OldConf, Name0 =/= Name]. +%% internal functions \ No newline at end of file diff --git a/apps/emqx_bridge/src/emqx_bridge_monitor.erl b/apps/emqx_bridge/src/emqx_bridge_monitor.erl index d76af5fb9..3136a74c9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_monitor.erl +++ b/apps/emqx_bridge/src/emqx_bridge_monitor.erl @@ -75,7 +75,7 @@ load_bridges(Configs) -> %% emqx_resource:check_and_create_local(ResourceId, ResourceType, Config, #{keep_retry => true}). load_bridge(Name, Type, Config) -> case emqx_resource:create_local( - emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_id(Type, Name), emqx_bridge:resource_type(Type), Config) of {ok, already_created} -> ok; {ok, _} -> ok; diff --git a/rebar.config b/rebar.config index 7210f11b0..7c5edc680 100644 --- a/rebar.config +++ b/rebar.config @@ -47,11 +47,11 @@ , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.9"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} - , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} + , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.2"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.3"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, "0.3.3"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} @@ -60,7 +60,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From 8730a03ab8d0160c9e77bd62828c736a203366f8 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 15 Sep 2021 10:15:21 +0800 Subject: [PATCH 28/84] feat(bridges): add start/stop/restart HTTP APIs for bridges --- apps/emqx_bridge/src/emqx_bridge_api.erl | 36 +++++++++++++++++-- apps/emqx_bridge/src/emqx_bridge_schema.erl | 5 +-- .../src/emqx_connector_mqtt.erl | 27 +++++++------- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 56c421e0c..e4805d7eb 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -23,6 +23,7 @@ , list_local_bridges/1 , crud_bridges_cluster/2 , crud_bridges/3 + , manage_bridges/2 ]). -define(TYPES, [mqtt]). @@ -113,14 +114,24 @@ crud_bridges_apis() -> operation_apis() -> Metadata = #{ post => #{ - description => <<"Restart bridges on all nodes in the cluster">>, + description => <<"Start/Stop/Restart bridges on a specific node">>, parameters => [ + param_path_node(), param_path_id(), param_path_operation()], responses => #{ <<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>, ['INTERNAL_ERROR']), <<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}}, - {"/bridges/:id/operation/:operation", Metadata, manage_bridges}. + {"/nodes/:node/bridges/:id/operation/:operation", Metadata, manage_bridges}. + +param_path_node() -> + #{ + name => node, + in => path, + schema => #{type => string}, + required => true, + example => node() + }. param_path_id() -> #{ @@ -192,6 +203,20 @@ crud_bridges(_, delete, #{bindings := #{id := Id}}) -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} end). +manage_bridges(post, #{bindings := #{node := Node, id := Id, operation := Op}}) -> + OperFun = + fun (<<"start">>) -> start_bridge; + (<<"stop">>) -> stop_bridge; + (<<"restart">>) -> restart_bridge + end, + ?TRY_PARSE_ID(Id, + case rpc_call(binary_to_atom(Node, latin1), emqx_bridge, OperFun(Op), + [BridgeType, BridgeName]) of + ok -> {200}; + {error, Reason} -> + {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} + end). + format_resp(#{id := Id, raw_config := RawConf, resource_data := #{mod := Mod, status := Status}}) -> IsConnected = fun(started) -> true; (_) -> false end, RawConf#{ @@ -202,7 +227,12 @@ format_resp(#{id := Id, raw_config := RawConf, resource_data := #{mod := Mod, st }. rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of + rpc_call(Node, ?MODULE, Fun, Args). + +rpc_call(Node, Mod, Fun, Args) when Node =:= node() -> + apply(Mod, Fun, Args); +rpc_call(Node, Mod, Fun, Args) -> + case rpc:call(Node, Mod, Fun, Args) of {badrpc, Reason} -> {error, Reason}; Res -> Res end. diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index beb0f282c..94cdaa30b 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -8,10 +8,7 @@ roots() -> ["bridges"]. fields("bridges") -> - [{mqtt, hoconsc:ref(?MODULE, "mqtt")}]; - -fields("mqtt") -> - [{"$name", hoconsc:ref(?MODULE, "mqtt_bridge")}]; + [{mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))}]; fields("mqtt_bridge") -> emqx_connector_mqtt:fields("config"). diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 708bcdeb9..55a4dde25 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -101,20 +101,21 @@ on_start(InstId, Conf) -> end end, InitRes, InOutConfigs). -on_stop(InstId, #{}) -> +on_stop(InstId, #{sub_bridges := NameList}) -> logger:info("stopping mqtt connector: ~p", [InstId]), - case ?MODULE:drop_bridge(InstId) of - ok -> ok; - {error, not_found} -> ok; - {error, Reason} -> - logger:error("stop bridge failed, error: ~p", [Reason]) - end. + lists:foreach(fun(Name) -> + case ?MODULE:drop_bridge(Name) of + ok -> ok; + {error, not_found} -> ok; + {error, Reason} -> + logger:error("stop channel ~p failed, error: ~p", [Name, Reason]) + end + end, NameList). %% TODO: let the emqx_resource trigger on_query/4 automatically according to the %% `message_in` and `message_out` config -on_query(InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, +on_query(_InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, baisc_conf := BasicConf}) -> - logger:debug("create channel to connector: ~p, conf: ~p", [InstId, Conf]), create_channel(Conf, Prefix, BasicConf); on_query(InstId, {publish_to_local, Msg}, _AfterQuery, _State) -> logger:debug("publish to local node, connector: ~p, msg: ~p", [InstId, Msg]); @@ -139,14 +140,16 @@ check_channel_id_dup(Confs) -> %% this is an `message_in` bridge create_channel(#{subscribe_remote_topic := _, id := Id} = InConf, NamePrefix, BasicConf) -> - logger:info("creating 'message_in' channel for: ~p", [Id]), + Name = bridge_name(NamePrefix, Id), + logger:info("creating 'message_in' channel ~p", [Name]), create_sub_bridge(BasicConf#{ - name => bridge_name(NamePrefix, Id), + name => Name, clientid => clientid(Id), subscriptions => InConf, forwards => undefined}); %% this is an `message_out` bridge create_channel(#{subscribe_local_topic := _, id := Id} = OutConf, NamePrefix, BasicConf) -> - logger:info("creating 'message_out' channel for: ~p", [Id]), + Name = bridge_name(NamePrefix, Id), + logger:info("creating 'message_out' channel ~p", [Name]), create_sub_bridge(BasicConf#{ name => bridge_name(NamePrefix, Id), clientid => clientid(Id), From c1ff8778e1e8622d0ae05a3d22a528038fb97ed9 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 15 Sep 2021 10:46:20 +0800 Subject: [PATCH 29/84] fix(bridges): add logs for creating/removing bridges --- apps/emqx_bridge/src/emqx_bridge.erl | 8 +++-- .../src/emqx_connector_mqtt.erl | 30 ++++++++++++------- .../src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 4d2b80ba7..badf97515 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -137,6 +137,7 @@ restart_bridge(Type, Name) -> emqx_resource:restart(resource_id(Type, Name)). create_bridge(Type, Name, Conf) -> + logger:info("create ~p bridge ~p use config: ~p", [Type, Name, Conf]), ResId = resource_id(Type, Name), case emqx_resource:create(ResId, emqx_bridge:resource_type(Type), Conf) of @@ -157,10 +158,12 @@ update_bridge(Type, Name, Conf) -> %% `message_out` are changed, then we should not restart the bridge, we only restart/start %% the channels. %% + logger:info("update ~p bridge ~p use config: ~p", [Type, Name, Conf]), emqx_resource:recreate(resource_id(Type, Name), emqx_bridge:resource_type(Type), Conf). remove_bridge(Type, Name, _Conf) -> + logger:info("remove ~p bridge ~p", [Type, Name]), case emqx_resource:remove(resource_id(Type, Name)) of ok -> ok; {error, not_found} -> ok; @@ -174,8 +177,9 @@ diff_confs(NewConfs, OldConfs) -> flatten_confs(Conf0) -> maps:from_list( - lists:append([do_flatten_confs(Type, Conf) - || {Type, Conf} <- maps:to_list(Conf0)])). + lists:flatmap(fun({Type, Conf}) -> + do_flatten_confs(Type, Conf) + end, maps:to_list(Conf0))). do_flatten_confs(Type, Conf0) -> [{{Type, Name}, Conf} || {Name, Conf} <- maps:to_list(Conf0)]. diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 55a4dde25..4f4d3e5c8 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -104,12 +104,7 @@ on_start(InstId, Conf) -> on_stop(InstId, #{sub_bridges := NameList}) -> logger:info("stopping mqtt connector: ~p", [InstId]), lists:foreach(fun(Name) -> - case ?MODULE:drop_bridge(Name) of - ok -> ok; - {error, not_found} -> ok; - {error, Reason} -> - logger:error("stop channel ~p failed, error: ~p", [Name, Reason]) - end + remove_channel(Name) end, NameList). %% TODO: let the emqx_resource trigger on_query/4 automatically according to the @@ -139,22 +134,35 @@ check_channel_id_dup(Confs) -> Confs. %% this is an `message_in` bridge -create_channel(#{subscribe_remote_topic := _, id := Id} = InConf, NamePrefix, BasicConf) -> +create_channel(#{subscribe_remote_topic := RemoteT, local_topic := LocalT, id := Id} = InConf, + NamePrefix, BasicConf) -> Name = bridge_name(NamePrefix, Id), - logger:info("creating 'message_in' channel ~p", [Name]), + logger:info("creating 'message_in' channel ~p, remote ~s -> local ~s", + [Name, RemoteT, LocalT]), create_sub_bridge(BasicConf#{ name => Name, clientid => clientid(Id), subscriptions => InConf, forwards => undefined}); %% this is an `message_out` bridge -create_channel(#{subscribe_local_topic := _, id := Id} = OutConf, NamePrefix, BasicConf) -> +create_channel(#{subscribe_local_topic := LocalT, remote_topic := RemoteT, id := Id} = OutConf, + NamePrefix, BasicConf) -> Name = bridge_name(NamePrefix, Id), - logger:info("creating 'message_out' channel ~p", [Name]), + logger:info("creating 'message_out' channel ~p, local ~s -> remote ~s", + [Name, LocalT, RemoteT]), create_sub_bridge(BasicConf#{ name => bridge_name(NamePrefix, Id), clientid => clientid(Id), subscriptions => undefined, forwards => OutConf}). +remove_channel(BridgeName) -> + logger:info("removing channel ~p", [BridgeName]), + case ?MODULE:drop_bridge(BridgeName) of + ok -> ok; + {error, not_found} -> ok; + {error, Reason} -> + logger:error("stop channel ~p failed, error: ~p", [BridgeName, Reason]) + end. + create_sub_bridge(#{name := Name} = Conf) -> case ?MODULE:create_bridge(Conf) of {ok, _Pid} -> @@ -206,7 +214,7 @@ bridge_name(Prefix, Id) -> list_to_atom(str(Prefix) ++ ":" ++ str(Id)). clientid(Id) -> - list_to_binary(str(Id) ++ ":" ++ emqx_plugin_libs_id:gen(4)). + list_to_binary(str(Id) ++ ":" ++ emqx_plugin_libs_id:gen(16)). str(A) when is_atom(A) -> atom_to_list(A); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 6ced719df..bb505f98d 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -183,7 +183,7 @@ callback_mode() -> [state_functions]. %% @doc Config should be a map(). init(#{name := Name} = ConnectOpts) -> - ?LOG(info, "starting bridge worker for ~p", [Name]), + ?LOG(debug, "starting bridge worker for ~p", [Name]), erlang:process_flag(trap_exit, true), Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), State = init_state(ConnectOpts), From 7058b8376095a6bf49d7e49d88f62c8c3c619361 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 15 Sep 2021 11:11:59 +0800 Subject: [PATCH 30/84] refactor(bridge): rename message_in/out to ingress/egress_channels --- apps/emqx_bridge/etc/emqx_bridge.conf | 8 ++++---- apps/emqx_bridge/src/emqx_bridge.erl | 4 ++-- apps/emqx_connector/src/emqx_connector_mqtt.erl | 14 +++++++------- .../src/mqtt/emqx_connector_mqtt_schema.erl | 8 ++++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index c04abf82a..4eddcb320 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -25,8 +25,8 @@ bridges.mqtt.my_mqtt_bridge_to_aws { certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" } - ## we will create one MQTT connection for each element of the `message_in` - message_in: [{ + ## we will create one MQTT connection for each element of the `ingress_channels` + ingress_channels: [{ ## the `id` will be used as part of the clientid id = "pull_msgs_from_aws" subscribe_remote_topic = "aws/#" @@ -36,8 +36,8 @@ bridges.mqtt.my_mqtt_bridge_to_aws { qos = "${qos}" retain = "${retain}" }] - ## we will create one MQTT connection for each element of the `message_out` - message_out: [{ + ## we will create one MQTT connection for each element of the `egress_channels` + egress_channels: [{ ## the `id` will be used as part of the clientid id = "push_msgs_to_aws" subscribe_local_topic = "emqx/#" diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index badf97515..adc5592cd 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -154,8 +154,8 @@ update_bridge(Type, Name, Conf) -> %% %% - if the connection related configs like `username` is updated, we should restart/start %% or stop bridges according to the change. - %% - if the connection related configs are not update, but channel configs `message_in` or - %% `message_out` are changed, then we should not restart the bridge, we only restart/start + %% - if the connection related configs are not update, but channel configs `ingress_channels` or + %% `egress_channels` are changed, then we should not restart the bridge, we only restart/start %% the channels. %% logger:info("update ~p bridge ~p use config: ~p", [Type, Name, Conf]), diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 4f4d3e5c8..95d19d5e8 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -89,8 +89,8 @@ on_start(InstId, Conf) -> NamePrefix = binary_to_list(InstId), BasicConf = basic_config(Conf), InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, sub_bridges => []}}, - InOutConfigs = check_channel_id_dup(maps:get(message_in, Conf, []) - ++ maps:get(message_out, Conf, [])), + InOutConfigs = check_channel_id_dup(maps:get(ingress_channels, Conf, []) + ++ maps:get(egress_channels, Conf, [])), lists:foldl(fun (_InOutConf, {error, Reason}) -> {error, Reason}; @@ -108,7 +108,7 @@ on_stop(InstId, #{sub_bridges := NameList}) -> end, NameList). %% TODO: let the emqx_resource trigger on_query/4 automatically according to the -%% `message_in` and `message_out` config +%% `ingress_channels` and `egress_channels` config on_query(_InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, baisc_conf := BasicConf}) -> create_channel(Conf, Prefix, BasicConf); @@ -133,21 +133,21 @@ check_channel_id_dup(Confs) -> end, Confs), Confs. -%% this is an `message_in` bridge +%% this is an `ingress_channels` bridge create_channel(#{subscribe_remote_topic := RemoteT, local_topic := LocalT, id := Id} = InConf, NamePrefix, BasicConf) -> Name = bridge_name(NamePrefix, Id), - logger:info("creating 'message_in' channel ~p, remote ~s -> local ~s", + logger:info("creating ingress channel ~p, remote ~s -> local ~s", [Name, RemoteT, LocalT]), create_sub_bridge(BasicConf#{ name => Name, clientid => clientid(Id), subscriptions => InConf, forwards => undefined}); -%% this is an `message_out` bridge +%% this is an `egress_channels` bridge create_channel(#{subscribe_local_topic := LocalT, remote_topic := RemoteT, id := Id} = OutConf, NamePrefix, BasicConf) -> Name = bridge_name(NamePrefix, Id), - logger:info("creating 'message_out' channel ~p, local ~s -> remote ~s", + logger:info("creating egress channel ~p, local ~s -> remote ~s", [Name, LocalT, RemoteT]), create_sub_bridge(BasicConf#{ name => bridge_name(NamePrefix, Id), diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 184a8610c..28995b666 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -38,17 +38,17 @@ fields("config") -> , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} , {max_inflight, hoconsc:mk(integer(), #{default => 32})} , {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))} - , {message_in, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "message_in")), #{default => []})} - , {message_out, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "message_out")), #{default => []})} + , {ingress_channels, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "ingress_channels")), #{default => []})} + , {egress_channels, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "egress_channels")), #{default => []})} ] ++ emqx_connector_schema_lib:ssl_fields(); -fields("message_in") -> +fields("ingress_channels") -> [ {subscribe_remote_topic, #{type => binary(), nullable => false}} , {local_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} , {subscribe_qos, hoconsc:mk(qos(), #{default => 1})} ] ++ common_inout_confs(); -fields("message_out") -> +fields("egress_channels") -> [ {subscribe_local_topic, #{type => binary(), nullable => false}} , {remote_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} ] ++ common_inout_confs(); From b1999b22052e8a1090e1e97b0a8629adfc702cba Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 15 Sep 2021 11:39:32 +0800 Subject: [PATCH 31/84] fix(bridge): update bridge with new conf failed --- apps/emqx_bridge/etc/emqx_bridge.conf | 90 +++++++++---------- apps/emqx_bridge/src/emqx_bridge.erl | 4 +- .../src/emqx_connector_mqtt.erl | 2 +- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index 4eddcb320..fc050f4b8 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -2,48 +2,48 @@ ## EMQ X Bridge ##-------------------------------------------------------------------- -bridges.mqtt.my_mqtt_bridge_to_aws { - server = "127.0.0.1:1883" - proto_ver = "v4" - username = "username1" - password = "" - clean_start = true - keepalive = 300 - retry_interval = "30s" - max_inflight = 32 - reconnect_interval = "30s" - bridge_mode = true - replayq { - dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" - seg_bytes = "100MB" - offload = false - max_total_bytes = "1GB" - } - ssl { - enable = false - keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" - certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" - cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } - ## we will create one MQTT connection for each element of the `ingress_channels` - ingress_channels: [{ - ## the `id` will be used as part of the clientid - id = "pull_msgs_from_aws" - subscribe_remote_topic = "aws/#" - subscribe_qos = 1 - local_topic = "from_aws/${topic}" - payload = "${payload}" - qos = "${qos}" - retain = "${retain}" - }] - ## we will create one MQTT connection for each element of the `egress_channels` - egress_channels: [{ - ## the `id` will be used as part of the clientid - id = "push_msgs_to_aws" - subscribe_local_topic = "emqx/#" - remote_topic = "from_emqx/${topic}" - payload = "${payload}" - qos = 1 - retain = false - }] -} +#bridges.mqtt.my_mqtt_bridge_to_aws { +# server = "127.0.0.1:1883" +# proto_ver = "v4" +# username = "username1" +# password = "" +# clean_start = true +# keepalive = 300 +# retry_interval = "30s" +# max_inflight = 32 +# reconnect_interval = "30s" +# bridge_mode = true +# replayq { +# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" +# seg_bytes = "100MB" +# offload = false +# max_total_bytes = "1GB" +# } +# ssl { +# enable = false +# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" +# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# ## we will create one MQTT connection for each element of the `ingress_channels` +# ingress_channels: [{ +# ## the `id` will be used as part of the clientid +# id = "pull_msgs_from_aws" +# subscribe_remote_topic = "aws/#" +# subscribe_qos = 1 +# local_topic = "from_aws/${topic}" +# payload = "${payload}" +# qos = "${qos}" +# retain = "${retain}" +# }] +# ## we will create one MQTT connection for each element of the `egress_channels` +# egress_channels: [{ +# ## the `id` will be used as part of the clientid +# id = "push_msgs_to_aws" +# subscribe_local_topic = "emqx/#" +# remote_topic = "from_emqx/${topic}" +# payload = "${payload}" +# qos = 1 +# retain = false +# }] +#} diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index adc5592cd..e3458adca 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -149,7 +149,7 @@ create_bridge(Type, Name, Conf) -> {error, Reason} end. -update_bridge(Type, Name, Conf) -> +update_bridge(Type, Name, {_OldConf, Conf}) -> %% TODO: sometimes its not necessary to restart the bridge connection. %% %% - if the connection related configs like `username` is updated, we should restart/start @@ -160,7 +160,7 @@ update_bridge(Type, Name, Conf) -> %% logger:info("update ~p bridge ~p use config: ~p", [Type, Name, Conf]), emqx_resource:recreate(resource_id(Type, Name), - emqx_bridge:resource_type(Type), Conf). + emqx_bridge:resource_type(Type), Conf, []). remove_bridge(Type, Name, _Conf) -> logger:info("remove ~p bridge ~p", [Type, Name]), diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 95d19d5e8..914d26d94 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -168,7 +168,7 @@ create_sub_bridge(#{name := Name} = Conf) -> {ok, _Pid} -> start_sub_bridge(Name); {error, {already_started, _Pid}} -> - ok; + {ok, Name}; {error, Reason} -> {error, Reason} end. From 20d465245492495b5c015d766cdc61d55ffb6e86 Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Mon, 13 Sep 2021 15:02:26 +0200 Subject: [PATCH 32/84] feat(mqueue): Interleave messages with different priorities --- apps/emqx/src/emqx_mqueue.erl | 64 +++++++++++- apps/emqx/src/emqx_pqueue.erl | 10 +- apps/emqx/test/emqx_mqueue_SUITE.erl | 140 ++++++++++++++++++++++++++- 3 files changed, 206 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index d625209ca..259c5c428 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -93,6 +93,11 @@ -define(MAX_LEN_INFINITY, 0). -define(INFO_KEYS, [store_qos0, max_len, len, dropped]). +-record(shift_opts, { + multiplier :: non_neg_integer(), + base :: integer() + }). + -record(mqueue, { store_qos0 = false :: boolean(), max_len = ?MAX_LEN_INFINITY :: count(), @@ -100,7 +105,10 @@ dropped = 0 :: count(), p_table = ?NO_PRIORITY_TABLE :: p_table(), default_p = ?LOWEST_PRIORITY :: priority(), - q = ?PQUEUE:new() :: pq() + q = ?PQUEUE:new() :: pq(), + shift_opts :: #shift_opts{}, + last_prio :: non_neg_integer() | undefined, + p_credit :: non_neg_integer() | undefined }). -type(mqueue() :: #mqueue{}). @@ -114,7 +122,8 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) -> #mqueue{max_len = MaxLen, store_qos0 = QoS_0, p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE), - default_p = get_priority_opt(Opts) + default_p = get_priority_opt(Opts), + shift_opts = get_shift_opt(Opts) }. -spec(info(mqueue()) -> emqx_types:infos()). @@ -173,9 +182,24 @@ in(Msg = #message{topic = Topic}, MQ = #mqueue{default_p = Dp, out(MQ = #mqueue{len = 0, q = Q}) -> 0 = ?PQUEUE:len(Q), %% assert, in this case, ?PQUEUE:len should be very cheap {empty, MQ}; -out(MQ = #mqueue{q = Q, len = Len}) -> +out(MQ = #mqueue{q = Q, len = Len, last_prio = undefined, shift_opts = ShiftOpts}) -> + {{value, Val, Prio}, Q1} = ?PQUEUE:out_p(Q), %% Shouldn't fail, since we've checked the length + MQ1 = MQ#mqueue{ + q = Q1, + len = Len - 1, + last_prio = Prio, + p_credit = get_credits(Prio, ShiftOpts) + }, + {{value, Val}, MQ1}; +out(MQ = #mqueue{q = Q, p_credit = 0}) -> + MQ1 = MQ#mqueue{ + q = ?PQUEUE:shift(Q), + last_prio = undefined + }, + out(MQ1); +out(MQ = #mqueue{q = Q, len = Len, p_credit = Cnt}) -> {R, Q1} = ?PQUEUE:out(Q), - {R, MQ#mqueue{q = Q1, len = Len - 1}}. + {R, MQ#mqueue{q = Q1, len = Len - 1, p_credit = Cnt - 1}}. get_opt(Key, Opts, Default) -> case maps:get(Key, Opts, Default) of @@ -196,3 +220,35 @@ get_priority_opt(Opts) -> %% while the highest 'infinity' is a [{infinity, queue:queue()}] get_priority(_Topic, ?NO_PRIORITY_TABLE, _) -> ?LOWEST_PRIORITY; get_priority(Topic, PTab, Dp) -> maps:get(Topic, PTab, Dp). + +get_credits(?HIGHEST_PRIORITY, Opts) -> + Infinity = 1000000, + get_credits(Infinity, Opts); +get_credits(Prio, #shift_opts{multiplier = Mult, base = Base}) -> + (Prio + Base + 1) * Mult - 1. + +get_shift_opt(Opts) -> + %% Using 10 as a multiplier by default. This is needed to minimize + %% overhead of ?PQUEUE:rotate + Mult = maps:get(shift_multiplier, Opts, 10), + true = is_integer(Mult) andalso Mult > 0, + Min = case Opts of + #{p_table := PTab} -> + case maps:size(PTab) of + 0 -> 0; + _ -> lists:min(maps:values(PTab)) + end; + _ -> + ?LOWEST_PRIORITY + end, + %% `mqueue' module supports negative priorities, but we don't want + %% the counter to be negative, so all priorities should be shifted + %% by a constant, if negative priorities are used: + Base = case Min < 0 of + true -> -Min; + false -> 0 + end, + #shift_opts{ + multiplier = Mult, + base = Base + }. diff --git a/apps/emqx/src/emqx_pqueue.erl b/apps/emqx/src/emqx_pqueue.erl index 85c89866d..5dd81af0b 100644 --- a/apps/emqx/src/emqx_pqueue.erl +++ b/apps/emqx/src/emqx_pqueue.erl @@ -55,6 +55,7 @@ , filter/2 , fold/3 , highest/1 + , shift/1 ]). -export_type([q/0]). @@ -170,6 +171,14 @@ out({pqueue, [{P, Q} | Queues]}) -> end, {R, NewQ}. +-spec(shift(pqueue()) -> pqueue()). +shift(Q = {queue, _, _, _}) -> + Q; +shift({pqueue, []}) -> + {pqueue, []}; %% Shouldn't happen? +shift({pqueue, [Hd|Rest]}) -> + {pqueue, Rest ++ [Hd]}. %% Let's hope there are not many priorities. + -spec(out_p(pqueue()) -> {empty | {value, any(), priority()}, pqueue()}). out_p({queue, _, _, _} = Q) -> add_p(out(Q), 0); out_p({pqueue, [{P, _} | _]} = Q) -> add_p(out(Q), maybe_negate_priority(P)). @@ -266,4 +275,3 @@ r2f([X,Y|R], L) -> {queue, [X,Y], lists:reverse(R, []), L}. maybe_negate_priority(infinity) -> infinity; maybe_negate_priority(P) -> -P. - diff --git a/apps/emqx/test/emqx_mqueue_SUITE.erl b/apps/emqx/test/emqx_mqueue_SUITE.erl index 34e509145..5b94ecbbf 100644 --- a/apps/emqx/test/emqx_mqueue_SUITE.erl +++ b/apps/emqx/test/emqx_mqueue_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(Q, emqx_mqueue). @@ -120,9 +121,88 @@ t_priority_mqueue(_) -> ?assertEqual(5, ?Q:len(Q5)), {_, Q6} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q5), ?assertEqual(5, ?Q:len(Q6)), - {{value, Msg}, Q7} = ?Q:out(Q6), - ?assertEqual(4, ?Q:len(Q7)), - ?assertEqual(<<"t3">>, Msg#message.topic). + {{value, _Msg}, Q7} = ?Q:out(Q6), + ?assertEqual(4, ?Q:len(Q7)). + +t_priority_mqueue_conservation(_) -> + true = proper:quickcheck(conservation_prop()). + +t_priority_order(_) -> + Opts = #{max_len => 5, + shift_multiplier => 1, + priorities => + #{<<"t1">> => 0, + <<"t2">> => 1, + <<"t3">> => 2 + }, + store_qos0 => false + }, + Messages = [{Topic, Message} || + Topic <- [<<"t1">>, <<"t2">>, <<"t3">>], + Message <- lists:seq(1, 10)], + Q = lists:foldl(fun({Topic, Message}, Q) -> + element(2, ?Q:in(#message{topic = Topic, qos = 1, payload = Message}, Q)) + end, + ?Q:init(Opts), + Messages), + ?assertMatch([{<<"t3">>, 6}, + {<<"t3">>, 7}, + {<<"t3">>, 8}, + + {<<"t2">>, 6}, + {<<"t2">>, 7}, + + {<<"t1">>, 6}, + + {<<"t3">>, 9}, + {<<"t3">>, 10}, + + {<<"t2">>, 8}, + + %% Note: for performance reasons we don't reset the + %% counter when we run out of messages with the + %% current prio, so next is t1: + {<<"t1">>, 7}, + + {<<"t2">>, 9}, + {<<"t2">>, 10}, + + {<<"t1">>, 8}, + {<<"t1">>, 9}, + {<<"t1">>, 10} + ], drain(Q)). + +t_priority_order2(_) -> + Opts = #{max_len => 5, + shift_multiplier => 2, + priorities => + #{<<"t1">> => 0, + <<"t2">> => 1 + }, + store_qos0 => false + }, + Messages = [{Topic, Message} || + Topic <- [<<"t1">>, <<"t2">>], + Message <- lists:seq(1, 10)], + Q = lists:foldl(fun({Topic, Message}, Q) -> + element(2, ?Q:in(#message{topic = Topic, qos = 1, payload = Message}, Q)) + end, + ?Q:init(Opts), + Messages), + ?assertMatch([{<<"t2">>, 6}, + {<<"t2">>, 7}, + {<<"t2">>, 8}, + {<<"t2">>, 9}, + + {<<"t1">>, 6}, + {<<"t1">>, 7}, + + {<<"t2">>, 10}, + + {<<"t1">>, 8}, + {<<"t1">>, 9}, + {<<"t1">>, 10} + ], drain(Q)). t_infinity_priority_mqueue(_) -> Opts = #{max_len => 0, @@ -163,3 +243,57 @@ t_dropped(_) -> {Msg, Q2} = ?Q:in(Msg, Q1), ?assertEqual(1, ?Q:dropped(Q2)). +conservation_prop() -> + ?FORALL({Priorities, Messages}, + ?LET(Priorities, topic_priorities(), + {Priorities, messages(Priorities)}), + try + Opts = #{max_len => 0, + priorities => maps:from_list(Priorities), + store_qos0 => false}, + %% Put messages in + Q1 = lists:foldl(fun({Topic, Message}, Q) -> + element(2, ?Q:in(#message{topic = Topic, qos = 1, payload = Message}, Q)) + end, + ?Q:init(Opts), + Messages), + %% Collect messages + Got = lists:sort(drain(Q1)), + Expected = lists:sort(Messages), + case Expected =:= Got of + true -> + true; + false -> + ct:pal("Mismatch: expected ~p~nGot ~p~n", [Expected, Got]), + false + end + catch + EC:Err:Stack -> + ct:pal("Error: ~p", [{EC, Err, Stack}]), + false + end). + +%% Proper generators: + +topic(Priorities) -> + {Topics, _} = lists:unzip(Priorities), + oneof(Topics). + +topic_priorities() -> + non_empty(list({binary(), priority()})). + +priority() -> + oneof([integer(), infinity]). + +messages(Topics) -> + list({topic(Topics), binary()}). + +%% Internal functions: + +drain(Q) -> + case ?Q:out(Q) of + {empty, _} -> + []; + {{value, #message{topic = T, payload = P}}, Q1} -> + [{T, P}|drain(Q1)] + end. From 3275ed4804bd01a2e72e20031b2138fd8ecfecb2 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 15 Sep 2021 18:14:05 +0800 Subject: [PATCH 33/84] fix(esasl): load esasl code --- rebar.config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config.erl b/rebar.config.erl index 3f4d86f37..f818b4f84 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -260,6 +260,7 @@ relx_apps(ReleaseType) -> , {mnesia, load} , {ekka, load} , {emqx_plugin_libs, load} + , {esasl, load} , observer_cli , emqx_http_lib , emqx_resource From 60e815fb9ab66f77bfd7fc0cd014e03860cd25cf Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Wed, 15 Sep 2021 15:27:25 +0800 Subject: [PATCH 34/84] fix: topic rewrite action type add all --- apps/emqx_modules/etc/emqx_modules.conf | 12 ++++++++++++ apps/emqx_modules/src/emqx_modules_schema.erl | 2 +- apps/emqx_modules/src/emqx_rewrite.erl | 4 +++- apps/emqx_modules/src/emqx_rewrite_api.erl | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/emqx_modules/etc/emqx_modules.conf b/apps/emqx_modules/etc/emqx_modules.conf index 56970bb0e..be34479e8 100644 --- a/apps/emqx_modules/etc/emqx_modules.conf +++ b/apps/emqx_modules/etc/emqx_modules.conf @@ -33,6 +33,18 @@ rewrite: [ # source_topic = "x/#" # re = "^x/y/(.+)$" # dest_topic = "z/y/$1" + # }, + # { + # action = subscribe + # source_topic = "x1/#" + # re = "^x1/y/(.+)$" + # dest_topic = "z1/y/$1" + # }, + # { + # action = all + # source_topic = "x2/#" + # re = "^x2/y/(.+)$" + # dest_topic = "z2/y/$1" # } ] diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index 15f6ab901..c989ecbed 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -45,7 +45,7 @@ fields("delayed") -> ]; fields("rewrite") -> - [ {action, hoconsc:enum([publish, subscribe])} + [ {action, hoconsc:enum([publish, subscribe, all])} , {source_topic, sc(binary(), #{})} , {re, sc(binary(), #{})} , {dest_topic, sc(binary(), #{})} diff --git a/apps/emqx_modules/src/emqx_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl index ae83339da..1b057ca51 100644 --- a/apps/emqx_modules/src/emqx_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -95,7 +95,9 @@ compile(Rules) -> publish -> {[{Topic, MP, Dest} | Acc1], Acc2}; subscribe -> - {Acc1, [{Topic, MP, Dest} | Acc2]} + {Acc1, [{Topic, MP, Dest} | Acc2]}; + all -> + {[{Topic, MP, Dest} | Acc1], [{Topic, MP, Dest} | Acc2]} end end, {[], []}, Rules). diff --git a/apps/emqx_modules/src/emqx_rewrite_api.erl b/apps/emqx_modules/src/emqx_rewrite_api.erl index 9f9b588b4..54defa599 100644 --- a/apps/emqx_modules/src/emqx_rewrite_api.erl +++ b/apps/emqx_modules/src/emqx_rewrite_api.erl @@ -35,7 +35,7 @@ api_spec() -> {[rewrite_api()], []}. properties() -> - properties([{action, string, <<"Node">>, [subscribe, publish]}, + properties([{action, string, <<"Action">>, [subscribe, publish, all]}, {source_topic, string, <<"Topic">>}, {re, string, <<"Regular expressions">>}, {dest_topic, string, <<"Destination topic">>}]). From 4a4014855f6cd006cfaba949befd38a4ce13c502 Mon Sep 17 00:00:00 2001 From: xujun540 <17683768715@163.com> Date: Wed, 15 Sep 2021 17:20:04 +0800 Subject: [PATCH 35/84] chore(autotest): add api test script --- .github/workflows/run_api_tests.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 57037616c..25f67d916 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -2,9 +2,9 @@ name: API Test Suite on: push: - tags: - - e* - - v* + tags: + - e* + - v* pull_request: jobs: @@ -13,6 +13,8 @@ jobs: container: "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" steps: - uses: actions/checkout@v2 + with: + repository: emqx/emqx - name: zip emqx-broker if: endsWith(github.repository, 'emqx') run: | @@ -39,6 +41,10 @@ jobs: - api_clients - api_routes - api_publish + - api_user + - api_login + - api_banned + - api_alarms steps: - uses: actions/checkout@v2 with: @@ -90,11 +96,6 @@ jobs: with: name: jmeter_logs path: ./jmeter_logs - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: jmeter_logs - path: emqx/log delete-package: runs-on: ubuntu-20.04 needs: api-test From a673f18c4107eca7e80a84fca2c782ee0a09931d Mon Sep 17 00:00:00 2001 From: xujun540 <17683768715@163.com> Date: Thu, 16 Sep 2021 09:55:48 +0800 Subject: [PATCH 36/84] chore(test): modify format --- .github/workflows/run_api_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 25f67d916..1826760f1 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -2,9 +2,9 @@ name: API Test Suite on: push: - tags: - - e* - - v* + tags: + - e* + - v* pull_request: jobs: From a45cbf504be37e8e7bbe1045085854866f4410fc Mon Sep 17 00:00:00 2001 From: xujun540 <17683768715@163.com> Date: Thu, 16 Sep 2021 10:38:59 +0800 Subject: [PATCH 37/84] chore(test): modify format --- .github/workflows/run_api_tests.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 1826760f1..4572f361a 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -13,8 +13,6 @@ jobs: container: "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" steps: - uses: actions/checkout@v2 - with: - repository: emqx/emqx - name: zip emqx-broker if: endsWith(github.repository, 'emqx') run: | From ca4b1ca3b58f1b83583c7c809c4364db0fbf30d8 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 14 Sep 2021 16:38:39 +0800 Subject: [PATCH 38/84] fix(auth mnesia api): fix the issue of missing line breaks in multi-line rules --- apps/emqx_authz/src/emqx_authz_api_sources.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 19d229dc2..cad968bd6 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -302,7 +302,7 @@ sources(get, _) -> {ok, Rules} -> lists:append(AccIn, [#{type => file, enable => Enable, - rules => iolist_to_binary([io_lib:format("~p.", [R]) || R <- Rules]), + rules => iolist_to_binary([io_lib:format("~p.\n", [R]) || R <- Rules]), annotations => #{status => healthy} }]); {error, _} -> @@ -360,7 +360,7 @@ source(get, #{bindings := #{type := Type}}) -> {ok, Rules} -> {200, #{type => file, enable => Enable, - rules => iolist_to_binary([io_lib:format("~p.", [R]) || R <- Rules]), + rules => iolist_to_binary([io_lib:format("~p.\n", [R]) || R <- Rules]), annotations => #{status => healthy} } }; From 129a171de209e8d459dd6dec089890b173234f34 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 14 Sep 2021 18:09:47 +0800 Subject: [PATCH 39/84] chore(authz): api returns the original content of the file Signed-off-by: zhanghongtong --- apps/emqx_authz/src/emqx_authz.app.src | 1 + .../emqx_authz/src/emqx_authz_api_sources.erl | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index f79e10f85..0cb714b94 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -6,6 +6,7 @@ {applications, [kernel, stdlib, + crypto, emqx_connector ]}, {env,[]}, diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index cad968bd6..a94a0e6c4 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -298,11 +298,11 @@ move_source_api() -> sources(get, _) -> Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) -> - case file:consult(Path) of + case file:read_file(Path) of {ok, Rules} -> lists:append(AccIn, [#{type => file, enable => Enable, - rules => iolist_to_binary([io_lib:format("~p.\n", [R]) || R <- Rules]), + rules => Rules, annotations => #{status => healthy} }]); {error, _} -> @@ -356,11 +356,11 @@ source(get, #{bindings := #{type := Type}}) -> case emqx_authz:lookup(Type) of {error, Reason} -> {404, #{message => atom_to_binary(Reason)}}; #{type := file, enable := Enable, path := Path}-> - case file:consult(Path) of + case file:read_file(Path) of {ok, Rules} -> {200, #{type => file, enable => Enable, - rules => iolist_to_binary([io_lib:format("~p.\n", [R]) || R <- Rules]), + rules => Rules, annotations => #{status => healthy} } }; @@ -476,8 +476,18 @@ write_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> }; write_cert(Source) -> Source. -write_file(Filename, Bytes) -> +write_file(Filename, Bytes0) -> ok = filelib:ensure_dir(Filename), + case file:read_file(Filename) of + {ok, Bytes1} -> + case crypto:hash(md5, Bytes1) =:= crypto:hash(md5, Bytes0) of + true -> {ok,iolist_to_binary(Filename)}; + false -> do_write_file(Filename, Bytes0) + end; + _ -> do_write_file(Filename, Bytes0) + end. + +do_write_file(Filename, Bytes) -> case file:write_file(Filename, Bytes) of ok -> {ok, iolist_to_binary(Filename)}; {error, Reason} -> From 834a6880626720e21e9c87da9d65d846d9cecb5d Mon Sep 17 00:00:00 2001 From: Jim Moen Date: Mon, 13 Sep 2021 20:07:18 +0800 Subject: [PATCH 40/84] fix(emqx_mgmt): clients api times using rfc3339. --- .../src/emqx_mgmt_api_clients.erl | 89 +++++++++++++++---- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 4b558eaae..45c4fdc36 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -33,14 +33,24 @@ , authz_cache/2 , subscribe/2 , unsubscribe/2 - , subscribe_batch/2]). + , subscribe_batch/2 + ]). -export([ query/4 - , format_channel_info/1]). + , format_channel_info/1 + ]). %% for batch operation -export([do_subscribe/3]). +%% for test suite +-export([ unix_ts_to_rfc3339_bin/1 + , unix_ts_to_rfc3339_bin/2 + , rfc3339_to_unix_ts_int/1 + , rfc3339_to_unix_ts_int/2 + ]). + + -define(CLIENT_QS_SCHEMA, {emqx_channel_info, [ {<<"node">>, atom} , {<<"username">>, binary} @@ -228,28 +238,28 @@ clients_api() -> name => gte_created_at, in => query, required => false, - description => <<"Search client session creation time by less than or equal method">>, + description => <<"Search client session creation time by greater than or equal method, rfc3339">>, schema => #{type => string} }, #{ name => lte_created_at, in => query, required => false, - description => <<"Search client session creation time by greater than or equal method">>, + description => <<"Search client session creation time by less than or equal method, rfc3339">>, schema => #{type => string} }, #{ name => gte_connected_at, in => query, required => false, - description => <<"Search client connection creation time by less than or equal method">>, + description => <<"Search client connection creation time by greater than or equal method, rfc3339">>, schema => #{type => string} }, #{ name => lte_connected_at, in => query, required => false, - description => <<"Search client connection creation time by greater than or equal method">>, + description => <<"Search client connection creation time by less than or equal method, rfc3339">>, schema => #{type => string} } ], @@ -376,7 +386,7 @@ subscribe_api() -> %%%============================================================================================== %% parameters trans clients(get, #{query_string := Qs}) -> - list(Qs). + list(generate_qs(Qs)). client(get, #{bindings := Bindings}) -> lookup(Bindings); @@ -520,6 +530,25 @@ do_unsubscribe(ClientID, Topic) -> Res -> Res end. + +%%-------------------------------------------------------------------- +%% QueryString Generation (rfc3339 to timestamp) + +generate_qs(Qs) -> + TimeKeys = [ <<"gte_created_at">> + , <<"lte_created_at">> + , <<"gte_connected_at">> + , <<"lte_connected_at">>], + Fun = + fun + (Key, NQs) -> + case NQs of + #{Key := Rfc3339Time} -> NQs#{Key => rfc3339_to_unix_ts_int(Rfc3339Time)}; + #{} -> NQs + end + end, + lists:foldl(Fun, Qs, TimeKeys). + %%-------------------------------------------------------------------- %% Query Functions @@ -613,24 +642,17 @@ run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) - %% format funcs format_channel_info({_, ClientInfo, ClientStats}) -> - Fun = - fun - (_Key, Value, Current) when is_map(Value) -> - maps:merge(Current, Value); - (Key, Value, Current) -> - maps:put(Key, Value, Current) - end, StatsMap = maps:without([memory, next_pkt_id, total_heap_size], maps:from_list(ClientStats)), - ClientInfoMap0 = maps:fold(Fun, #{}, ClientInfo), + ClientInfoMap0 = maps:fold(fun take_maps_from_inner/3, #{}, ClientInfo), IpAddress = peer_to_binary(maps:get(peername, ClientInfoMap0)), Connected = maps:get(conn_state, ClientInfoMap0) =:= connected, ClientInfoMap1 = maps:merge(StatsMap, ClientInfoMap0), ClientInfoMap2 = maps:put(node, node(), ClientInfoMap1), ClientInfoMap3 = maps:put(ip_address, IpAddress, ClientInfoMap2), ClientInfoMap = maps:put(connected, Connected, ClientInfoMap3), - RemoveList = [ - auth_result + RemoveList = + [ auth_result , peername , sockname , peerhost @@ -654,7 +676,23 @@ format_channel_info({_, ClientInfo, ClientStats}) -> , retry_interval , upgrade_qos ], - maps:without(RemoveList, ClientInfoMap). + TimesKeys = [created_at, connected_at, disconnected_at], + %% format timestamp to rfc3339 + lists:foldl(fun result_format_time_fun/2 + , maps:without(RemoveList, ClientInfoMap) + , TimesKeys). + +%% format func helpers +take_maps_from_inner(_Key, Value, Current) when is_map(Value) -> + maps:merge(Current, Value); +take_maps_from_inner(Key, Value, Current) -> + maps:put(Key, Value, Current). + +result_format_time_fun(Key, NClientInfoMap) -> + case NClientInfoMap of + #{Key := TimeStamp} -> NClientInfoMap#{Key => unix_ts_to_rfc3339_bin(TimeStamp)}; + #{} -> NClientInfoMap + end. peer_to_binary({Addr, Port}) -> AddrBinary = list_to_binary(inet:ntoa(Addr)), @@ -669,3 +707,18 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> result => AuthzResult, updated_time => Timestamp }. + +%%-------------------------------------------------------------------- +%% time format funcs + +unix_ts_to_rfc3339_bin(TimeStamp) -> + unix_ts_to_rfc3339_bin(TimeStamp, millisecond). + +unix_ts_to_rfc3339_bin(TimeStamp, Unit) when is_integer(TimeStamp) -> + list_to_binary(calendar:system_time_to_rfc3339(TimeStamp, [{unit, Unit}])). + +rfc3339_to_unix_ts_int(DateTime) -> + rfc3339_to_unix_ts_int(DateTime, millisecond). + +rfc3339_to_unix_ts_int(DateTime, Unit) when is_binary(DateTime) -> + calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}]). From 8cbec2ec6e1590e424c02b023b7caa77c391eb22 Mon Sep 17 00:00:00 2001 From: Jim Moen Date: Tue, 14 Sep 2021 16:13:41 +0800 Subject: [PATCH 41/84] chore(emqx_mgmt): test suite for clients api times using rfc3339. --- .../test/emqx_mgmt_clients_api_SUITE.erl | 75 ++++++++++++++++--- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl index 54b4f92ff..9735a7bcf 100644 --- a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -44,20 +44,20 @@ t_clients(_) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), - {ok, _} = emqtt:connect(C1), + {ok, _} = emqtt:connect(C1), {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), - {ok, _} = emqtt:connect(C2), + {ok, _} = emqtt:connect(C2), timer:sleep(300), %% get /clients - ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), - {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), ClientsResponse = emqx_json:decode(Clients, [return_maps]), - ClientsMeta = maps:get(<<"meta">>, ClientsResponse), - ClientsPage = maps:get(<<"page">>, ClientsMeta), - ClientsLimit = maps:get(<<"limit">>, ClientsMeta), - ClientsCount = maps:get(<<"count">>, ClientsMeta), + ClientsMeta = maps:get(<<"meta">>, ClientsResponse), + ClientsPage = maps:get(<<"page">>, ClientsMeta), + ClientsLimit = maps:get(<<"limit">>, ClientsMeta), + ClientsCount = maps:get(<<"count">>, ClientsMeta), ?assertEqual(ClientsPage, 1), ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()), ?assertEqual(ClientsCount, 2), @@ -73,8 +73,8 @@ t_clients(_) -> Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path), timer:sleep(300), - AfterKickoutResponse = emqx_mgmt_api_test_util:request_api(get, Client2Path), - ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse), + AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path), + ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2), %% get /clients/:clientid/authz_cache should has no authz cache Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "authz_cache"]), @@ -94,4 +94,57 @@ t_clients(_) -> UnSubscribePath = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1), "unsubscribe"]), {ok, _} = emqx_mgmt_api_test_util:request_api(post, UnSubscribePath, "", AuthHeader, SubscribeBody), timer:sleep(100), - ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)). + ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)), + + %% testcase cleanup, kickout client1 + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client1Path), + timer:sleep(300), + AfterKickoutResponse1 = emqx_mgmt_api_test_util:request_api(get, Client1Path), + ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse1). + +t_query_clients_with_time(_) -> + process_flag(trap_exit, true), + Username1 = <<"user1">>, + ClientId1 = <<"client1">>, + + Username2 = <<"user2">>, + ClientId2 = <<"client2">>, + + {ok, C1} = emqtt:start_link(#{username => Username1, clientid => ClientId1}), + {ok, _} = emqtt:connect(C1), + {ok, C2} = emqtt:start_link(#{username => Username2, clientid => ClientId2}), + {ok, _} = emqtt:connect(C2), + + timer:sleep(100), + + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + %% get /clients with time(rfc3339) + NowTimeStampInt = erlang:system_time(millisecond), + %% Do not uri_encode `=` to `%3D` + NowTimeString = emqx_http_lib:uri_encode(binary:bin_to_list(emqx_mgmt_api_clients:unix_ts_to_rfc3339_bin(NowTimeStampInt))), + Parameters = [Param ++ NowTimeString + || Param <- [ "lte_created_at=" + , "lte_connected_at=" + , "gte_created_at=" + , "gte_connected_at="]], + RequestResults = [emqx_mgmt_api_test_util:request_api(get, ClientsPath, Param, AuthHeader) + || Param <- Parameters], + DecodedResults = [emqx_json:decode(Response, [return_maps]) + || {ok, Response} <- RequestResults], + {LteResponseDecodeds, GteResponseDecodeds} = lists:split(2, DecodedResults), + %% EachData :: list() + [?assert( emqx_mgmt_api_clients:rfc3339_to_unix_ts_int(CreatedAt) < NowTimeStampInt) + || #{<<"data">> := EachData} <- LteResponseDecodeds, + #{<<"created_at">> := CreatedAt} <- EachData], + [?assert(emqx_mgmt_api_clients:rfc3339_to_unix_ts_int(ConnectedAt) < NowTimeStampInt) + || #{<<"data">> := EachData} <- LteResponseDecodeds, + #{<<"connected_at">> := ConnectedAt} <- EachData], + [?assertEqual(EachData, []) + || #{<<"data">> := EachData} <- GteResponseDecodeds], + + %% testcase cleanup, kickout client1 and client2 + Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), + Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client1Path), + {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path). From 294c1a5f690904d9a0d27294b50c32bdeb99706f Mon Sep 17 00:00:00 2001 From: Jim Moen Date: Thu, 16 Sep 2021 11:01:28 +0800 Subject: [PATCH 42/84] fix(emqx_mgmt): clients api query params supports epoch(milliseconds). --- .../src/emqx_mgmt_api_clients.erl | 54 +++++++++++-------- .../test/emqx_mgmt_clients_api_SUITE.erl | 25 +++++---- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 45c4fdc36..16d2d99af 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -46,8 +46,8 @@ %% for test suite -export([ unix_ts_to_rfc3339_bin/1 , unix_ts_to_rfc3339_bin/2 - , rfc3339_to_unix_ts_int/1 - , rfc3339_to_unix_ts_int/2 + , time_string_to_unix_ts_int/1 + , time_string_to_unix_ts_int/2 ]). @@ -106,9 +106,9 @@ properties(client) -> {clean_start, boolean, <<"Indicate whether the client is using a brand new session">>}, {clientid, string , <<"Client identifier">>}, {connected, boolean, <<"Whether the client is connected">>}, - {connected_at, string , <<"Client connection time">>}, - {created_at, string , <<"Session creation time">>}, - {disconnected_at, string , <<"Client offline time, This field is only valid and returned when connected is false">>}, + {connected_at, string , <<"Client connection time, rfc3339">>}, + {created_at, string , <<"Session creation time, rfc3339">>}, + {disconnected_at, string , <<"Client offline time, This field is only valid and returned when connected is false, rfc3339">>}, {expiry_interval, integer, <<"Session expiration interval, with the unit of second">>}, {heap_size, integer, <<"Process heap size with the unit of byte">>}, {inflight_cnt, integer, <<"Current length of inflight">>}, @@ -238,28 +238,28 @@ clients_api() -> name => gte_created_at, in => query, required => false, - description => <<"Search client session creation time by greater than or equal method, rfc3339">>, + description => <<"Search client session creation time by greater than or equal method, rfc3339 or timestamp(millisecond)">>, schema => #{type => string} }, #{ name => lte_created_at, in => query, required => false, - description => <<"Search client session creation time by less than or equal method, rfc3339">>, + description => <<"Search client session creation time by less than or equal method, rfc3339 or timestamp(millisecond)">>, schema => #{type => string} }, #{ name => gte_connected_at, in => query, required => false, - description => <<"Search client connection creation time by greater than or equal method, rfc3339">>, + description => <<"Search client connection creation time by greater than or equal method, rfc3339 or timestamp(millisecond)">>, schema => #{type => string} }, #{ name => lte_connected_at, in => query, required => false, - description => <<"Search client connection creation time by less than or equal method, rfc3339">>, + description => <<"Search client connection creation time by less than or equal method, rfc3339 or timestamp(millisecond) ">>, schema => #{type => string} } ], @@ -532,22 +532,25 @@ do_unsubscribe(ClientID, Topic) -> end. %%-------------------------------------------------------------------- -%% QueryString Generation (rfc3339 to timestamp) +%% QueryString Generation (try rfc3339 to timestamp or keep timestamp) + +time_keys() -> + [ <<"gte_created_at">> + , <<"lte_created_at">> + , <<"gte_connected_at">> + , <<"lte_connected_at">>]. generate_qs(Qs) -> - TimeKeys = [ <<"gte_created_at">> - , <<"lte_created_at">> - , <<"gte_connected_at">> - , <<"lte_connected_at">>], Fun = - fun - (Key, NQs) -> + fun (Key, NQs) -> case NQs of - #{Key := Rfc3339Time} -> NQs#{Key => rfc3339_to_unix_ts_int(Rfc3339Time)}; - #{} -> NQs + %% TimeString likes "2021-01-01T00:00:00.000+08:00" (in rfc3339) + %% or "1609430400000" (in millisecond) + #{Key := TimeString} -> NQs#{Key => time_string_to_unix_ts_int(TimeString)}; + #{} -> NQs end end, - lists:foldl(Fun, Qs, TimeKeys). + lists:foldl(Fun, Qs, time_keys()). %%-------------------------------------------------------------------- %% Query Functions @@ -717,8 +720,13 @@ unix_ts_to_rfc3339_bin(TimeStamp) -> unix_ts_to_rfc3339_bin(TimeStamp, Unit) when is_integer(TimeStamp) -> list_to_binary(calendar:system_time_to_rfc3339(TimeStamp, [{unit, Unit}])). -rfc3339_to_unix_ts_int(DateTime) -> - rfc3339_to_unix_ts_int(DateTime, millisecond). +time_string_to_unix_ts_int(DateTime) -> + time_string_to_unix_ts_int(DateTime, millisecond). -rfc3339_to_unix_ts_int(DateTime, Unit) when is_binary(DateTime) -> - calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}]). +time_string_to_unix_ts_int(DateTime, Unit) when is_binary(DateTime) -> + try binary_to_integer(DateTime) of + TimeStamp when is_integer(TimeStamp) -> TimeStamp + catch + error:badarg -> + calendar:rfc3339_to_system_time(binary_to_list(DateTime), [{unit, Unit}]) + end. diff --git a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl index 9735a7bcf..74838c91b 100644 --- a/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_clients_api_SUITE.erl @@ -104,6 +104,7 @@ t_clients(_) -> t_query_clients_with_time(_) -> process_flag(trap_exit, true), + Username1 = <<"user1">>, ClientId1 = <<"client1">>, @@ -122,22 +123,26 @@ t_query_clients_with_time(_) -> %% get /clients with time(rfc3339) NowTimeStampInt = erlang:system_time(millisecond), %% Do not uri_encode `=` to `%3D` - NowTimeString = emqx_http_lib:uri_encode(binary:bin_to_list(emqx_mgmt_api_clients:unix_ts_to_rfc3339_bin(NowTimeStampInt))), - Parameters = [Param ++ NowTimeString - || Param <- [ "lte_created_at=" - , "lte_connected_at=" - , "gte_created_at=" - , "gte_connected_at="]], + Rfc3339String = emqx_http_lib:uri_encode(binary:bin_to_list(emqx_mgmt_api_clients:unix_ts_to_rfc3339_bin(NowTimeStampInt))), + TimeStampString = emqx_http_lib:uri_encode(integer_to_list(NowTimeStampInt)), + + LteKeys = ["lte_created_at=", "lte_connected_at="], + GteKeys = ["gte_created_at=", "gte_connected_at="], + LteParamRfc3339 = [Param ++ Rfc3339String || Param <- LteKeys], + LteParamStamp = [Param ++ TimeStampString || Param <- LteKeys], + GteParamRfc3339 = [Param ++ Rfc3339String || Param <- GteKeys], + GteParamStamp = [Param ++ TimeStampString || Param <- GteKeys], + RequestResults = [emqx_mgmt_api_test_util:request_api(get, ClientsPath, Param, AuthHeader) - || Param <- Parameters], + || Param <- LteParamRfc3339 ++ LteParamStamp ++ GteParamRfc3339 ++ GteParamStamp], DecodedResults = [emqx_json:decode(Response, [return_maps]) || {ok, Response} <- RequestResults], - {LteResponseDecodeds, GteResponseDecodeds} = lists:split(2, DecodedResults), + {LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults), %% EachData :: list() - [?assert( emqx_mgmt_api_clients:rfc3339_to_unix_ts_int(CreatedAt) < NowTimeStampInt) + [?assert( emqx_mgmt_api_clients:time_string_to_unix_ts_int(CreatedAt) < NowTimeStampInt) || #{<<"data">> := EachData} <- LteResponseDecodeds, #{<<"created_at">> := CreatedAt} <- EachData], - [?assert(emqx_mgmt_api_clients:rfc3339_to_unix_ts_int(ConnectedAt) < NowTimeStampInt) + [?assert(emqx_mgmt_api_clients:time_string_to_unix_ts_int(ConnectedAt) < NowTimeStampInt) || #{<<"data">> := EachData} <- LteResponseDecodeds, #{<<"connected_at">> := ConnectedAt} <- EachData], [?assertEqual(EachData, []) From 1e8948e09196fd58dd13b0fd60ea806508fe21f8 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 13 Sep 2021 17:08:17 +0800 Subject: [PATCH 43/84] chore(CI): use ghcr Signed-off-by: zhanghongtong --- .github/workflows/build_packages.yaml | 16 +++---- .github/workflows/build_slim_packages.yaml | 17 ++++--- .github/workflows/check_deps_integrity.yaml | 7 +-- .github/workflows/run_api_tests.yaml | 5 +- .github/workflows/run_emqx_app_tests.yaml | 51 ++++++++++----------- .github/workflows/run_fvt_tests.yaml | 8 ++-- .github/workflows/run_relup_tests.yaml | 8 ++-- .github/workflows/run_test_cases.yaml | 16 +++---- deploy/docker/Dockerfile | 2 +- 9 files changed, 63 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 0c6d065a9..322d38206 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -12,12 +12,12 @@ jobs: prepare: strategy: matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" outputs: profiles: ${{ steps.set_profile.outputs.profiles }} @@ -306,7 +306,7 @@ jobs: done - name: build emqx packages env: - ERL_OTP: erl${{ matrix.otp }} + OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -316,7 +316,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - emqx/build-env:$ERL_OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder-helper/5.0:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -375,7 +375,7 @@ jobs: tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }} build-args: | PKG_VSN=${{ steps.version.outputs.version }} - BUILD_FROM=emqx/build-env:erl${{ matrix.otp }}-alpine + BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -395,7 +395,7 @@ jobs: tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }} build-args: | PKG_VSN=${{ steps.version.outputs.version }} - BUILD_FROM=emqx/build-env:erl${{ matrix.otp }}-alpine + BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 9578c6f9d..2fb447d26 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -14,14 +14,13 @@ jobs: strategy: matrix: - erl_otp: - - erl24.0.5-emqx-1 - + otp: + - 24.0.5-emqx-1 os: - ubuntu20.04 - centos7 - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v1 @@ -58,7 +57,7 @@ jobs: strategy: matrix: - erl_otp: + otp: - 24.0.5-emqx-1 steps: @@ -83,7 +82,7 @@ jobs: id: cache with: path: ~/.kerl - key: erl${{ matrix.erl_otp }}-macos10.15 + key: erl${{ matrix.otp }}-macos10.15 - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 @@ -92,11 +91,11 @@ jobs: OTP_GITHUB_URL: https://github.com/emqx/otp run: | kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + kerl build ${{ matrix.otp }} + kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate + . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 96cd71bf0..c316b582d 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -4,13 +4,8 @@ on: [pull_request] jobs: check_deps_integrity: - strategy: - matrix: - container: - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" - runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:24.0.5-emqx-1-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 4572f361a..af9be07e0 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -10,7 +10,8 @@ on: jobs: build: runs-on: ubuntu-latest - container: "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder-helper/5.0:23.2.7.2-emqx-2-ubuntu20.04" + steps: - uses: actions/checkout@v2 - name: zip emqx-broker @@ -27,6 +28,7 @@ jobs: with: name: emqx-broker path: _packages/**/*.zip + api-test: needs: build runs-on: ubuntu-latest @@ -94,6 +96,7 @@ jobs: with: name: jmeter_logs path: ./jmeter_logs + delete-package: runs-on: ubuntu-20.04 needs: api-test diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index d05d41969..2ce1edd8c 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -8,31 +8,30 @@ on: pull_request: jobs: - check_all: - strategy: - matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + check_all: + strategy: + matrix: + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" - runs-on: ubuntu-20.04 - container: ${{ matrix.container }} - - steps: - - uses: actions/checkout@v2 - - name: run - run: | - make ensure-rebar3 - cp rebar3 apps/emqx/ - cd apps/emqx - ./rebar3 xref - ./rebar3 dialyzer - ./rebar3 eunit -v - ./rebar3 ct -v - ./rebar3 proper -d test/props - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: apps/emqx/_build/test/logs + runs-on: ubuntu-20.04 + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" + steps: + - uses: actions/checkout@v2 + - name: run + run: | + make ensure-rebar3 + cp rebar3 apps/emqx/ + cd apps/emqx + ./rebar3 xref + ./rebar3 dialyzer + ./rebar3 eunit -v + ./rebar3 ct -v + ./rebar3 proper -d test/props + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: apps/emqx/_build/test/logs diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a4b9df5c2..8eade9127 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -11,12 +11,12 @@ jobs: prepare: strategy: matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" outputs: profile: ${{ steps.profile.outputs.profile }} diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 312ef1152..198b140ed 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -11,12 +11,12 @@ jobs: relup_test: strategy: matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" defaults: run: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index b97c3c003..5f631debb 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -11,12 +11,12 @@ jobs: run_static_analysis: strategy: matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" steps: - uses: actions/checkout@v2 @@ -34,12 +34,12 @@ jobs: run_proper_test: strategy: matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + otp: + - "23.2.7.2-emqx-2" + - "24.0.5-emqx-1" runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 623d01d96..5aa1d8864 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:24.0.5-emqx-1-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder From ba26a8511c5c4a9d758579a04801c27350d50c36 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 16 Sep 2021 14:27:11 +0800 Subject: [PATCH 44/84] fix(emqx_types): emqx_types:version() used but undefined. --- apps/emqx/src/emqx_frame.erl | 4 ++-- apps/emqx/src/emqx_packet.erl | 2 +- apps/emqx/src/emqx_types.erl | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 79a740bed..cea94eec8 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -44,7 +44,7 @@ -type(options() :: #{strict_mode => boolean(), max_size => 1..?MAX_PACKET_SIZE, - version => emqx_types:version() + version => emqx_types:proto_ver() }). -type(parse_state() :: {none, options()} | {cont_state(), options()}). @@ -490,7 +490,7 @@ serialize_pkt(Packet, #{version := Ver, max_size := MaxSize}) -> -spec(serialize(emqx_types:packet()) -> iodata()). serialize(Packet) -> serialize(Packet, ?MQTT_PROTO_V4). --spec(serialize(emqx_types:packet(), emqx_types:version()) -> iodata()). +-spec(serialize(emqx_types:packet(), emqx_types:proto_ver()) -> iodata()). serialize(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, Ver) -> diff --git a/apps/emqx/src/emqx_packet.erl b/apps/emqx/src/emqx_packet.erl index a4d440ba1..d1577e0c7 100644 --- a/apps/emqx/src/emqx_packet.erl +++ b/apps/emqx/src/emqx_packet.erl @@ -114,7 +114,7 @@ proto_name(#mqtt_packet_connect{proto_name = Name}) -> Name. %% @doc Protocol version of the CONNECT Packet. --spec(proto_ver(emqx_types:packet()|connect()) -> emqx_types:version()). +-spec(proto_ver(emqx_types:packet()|connect()) -> emqx_types:proto_ver()). proto_ver(?CONNECT_PACKET(ConnPkt)) -> proto_ver(ConnPkt); proto_ver(#mqtt_packet_connect{proto_ver = Ver}) -> diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 84868f473..940c9b630 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -20,7 +20,7 @@ -include("emqx_mqtt.hrl"). -include("types.hrl"). --export_type([ ver/0 +-export_type([ proto_ver/0 , qos/0 , qos_name/0 ]). @@ -91,11 +91,11 @@ -export_type([oom_policy/0]). --type(ver() :: ?MQTT_PROTO_V3 - | ?MQTT_PROTO_V4 - | ?MQTT_PROTO_V5 - | non_neg_integer() - | binary() % For lwm2m, mqtt-sn... +-type(proto_ver() :: ?MQTT_PROTO_V3 + | ?MQTT_PROTO_V4 + | ?MQTT_PROTO_V5 + | non_neg_integer() + | binary() % For lwm2m, mqtt-sn... ). -type(qos() :: ?QOS_0 | ?QOS_1 | ?QOS_2). @@ -116,7 +116,7 @@ peercert := nossl | undefined | esockd_peercert:peercert(), conn_mod := module(), proto_name => binary(), - proto_ver => ver(), + proto_ver => proto_ver(), clean_start => boolean(), clientid => clientid(), username => username(), @@ -187,12 +187,12 @@ -type(message() :: #message{}). -type(flag() :: sys | dup | retain | atom()). -type(flags() :: #{flag() := boolean()}). --type(headers() :: #{proto_ver => ver(), - protocol => protocol(), - username => username(), - peerhost => peerhost(), +-type(headers() :: #{proto_ver => proto_ver(), + protocol => protocol(), + username => username(), + peerhost => peerhost(), properties => properties(), - atom() => term()}). + atom() => term()}). -type(banned() :: #banned{}). -type(deliver() :: {deliver, topic(), message()}). From deac54c847fb01e1b55dd539193c3245cac5b5fa Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 16 Sep 2021 14:28:59 +0800 Subject: [PATCH 45/84] chore(for editor): Add comments for Emacs major mode. --- .ci/fvt_tests/http_server/src/http_server.app.src | 1 + apps/emqx/src/emqx.app.src | 1 + .../emqx_hocon_plugin/src/emqx_hocon_plugin.app.src | 1 + .../emqx_mini_plugin/src/emqx_mini_plugin.app.src | 1 + apps/emqx_authn/src/emqx_authn.app.src | 1 + apps/emqx_authz/src/emqx_authz.app.src | 1 + apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src | 1 + apps/emqx_bridge/src/emqx_bridge.app.src | 1 + apps/emqx_connector/src/emqx_connector.app.src | 1 + apps/emqx_dashboard/src/emqx_dashboard.app.src | 1 + apps/emqx_exhook/src/emqx_exhook.app.src | 1 + apps/emqx_exhook/src/emqx_exhook.appup.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 1 + apps/emqx_machine/src/emqx_machine.app.src | 1 + apps/emqx_management/src/emqx_management.app.src | 1 + apps/emqx_modules/src/emqx_modules.app.src | 1 + apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 1 + apps/emqx_prometheus/src/emqx_prometheus.app.src | 1 + apps/emqx_resource/elvis.config | 1 + apps/emqx_resource/src/emqx_resource.app.src | 1 + apps/emqx_retainer/src/emqx_retainer.app.src | 1 + apps/emqx_rule_actions/src/emqx_rule_actions.app.src | 1 + apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 1 + apps/emqx_statsd/src/emqx_statsd.app.src | 1 + elvis.config | 2 +- rebar.config | 2 +- 26 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.ci/fvt_tests/http_server/src/http_server.app.src b/.ci/fvt_tests/http_server/src/http_server.app.src index f351bb349..420aff9d3 100644 --- a/.ci/fvt_tests/http_server/src/http_server.app.src +++ b/.ci/fvt_tests/http_server/src/http_server.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, http_server, [{description, "An OTP application"}, {vsn, "0.1.0"}, diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 57f7cd57f..031e4f654 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx, [{id, "emqx"}, {description, "EMQ X Core"}, diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src index d09f01b1c..03cf8e69c 100644 --- a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_hocon_plugin/src/emqx_hocon_plugin.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_hocon_plugin, [{description, "An EMQ X plugin for hocon testcase"}, {vsn, "0.1"}, diff --git a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src index 413b4e1fd..0a1806f9c 100644 --- a/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src +++ b/apps/emqx/test/emqx_plugins_SUITE_data/emqx_mini_plugin/src/emqx_mini_plugin.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_mini_plugin, [{description, "An EMQ X plugin for testcase"}, {vsn, "0.1"}, diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 3b89d2a99..7aef92262 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_authn, [{description, "EMQ X Authentication"}, {vsn, "0.1.0"}, diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 0cb714b94..ff43b3536 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_authz, [{description, "An OTP application"}, {vsn, "0.1.1"}, diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index 0d87af87a..92d6932ca 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_auto_subscribe, [{description, "An OTP application"}, {vsn, "0.1.0"}, diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 9c0f6b779..2a2f11603 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_bridge, [{description, "An OTP application"}, {vsn, "0.1.0"}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index f4481dc2c..3e59d3528 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_connector, [{description, "An OTP application"}, {vsn, "0.1.1"}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 425180af4..5958d5f36 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, {vsn, "5.0.0"}, % strict semver, bump manually! diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index c306a5ea4..bfd33a661 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, {vsn, "5.0.0"}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 9e142d9e2..5ca8ba8da 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,4 +1,4 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- {VSN, [ {<<".*">>, []} diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 2fc329711..69aa9418e 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_gateway, [{description, "The Gateway management application"}, {vsn, "0.1.0"}, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 21da4a4c8..8140c7b6a 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_machine, [{id, "emqx_machine"}, {description, "The EMQ X Machine"}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index b859c2d55..1349f541c 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_management, [{description, "EMQ X Management API and CLI"}, {vsn, "5.0.0"}, % strict semver, bump manually! diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index cfa315c9b..06e8ecc1f 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_modules, [{description, "EMQ X Modules"}, {vsn, "5.0.0"}, diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 82937d033..f515244c1 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_plugin_libs, [{description, "EMQ X Plugin utility libs"}, {vsn, "4.3.1"}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index e1e55b599..8b5968ea0 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_prometheus, [{description, "Prometheus for EMQ X"}, {vsn, "5.0.0"}, % strict semver, bump manually! diff --git a/apps/emqx_resource/elvis.config b/apps/emqx_resource/elvis.config index 59aa13fbe..5a0ec61dd 100644 --- a/apps/emqx_resource/elvis.config +++ b/apps/emqx_resource/elvis.config @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- [{elvis, [{config, [ #{dirs => ["src"], diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 7e7ebe9ff..1b93aa0de 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_resource, [{description, "An OTP application"}, {vsn, "0.1.0"}, diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index b2023671b..482196b64 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_retainer, [{description, "EMQ X Retainer"}, {vsn, "5.0.0"}, % strict semver, bump manually! diff --git a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src index 8e7b66b55..8c2b8d247 100644 --- a/apps/emqx_rule_actions/src/emqx_rule_actions.app.src +++ b/apps/emqx_rule_actions/src/emqx_rule_actions.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_rule_actions, [{description, "Rule actions"}, {vsn, "5.0.0"}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index ff25dcfd3..a93a7e14d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, {vsn, "5.0.0"}, % strict semver, bump manually! diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 747c9447c..aa1c14ded 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_statsd, [{description, "An OTP application"}, {vsn, "5.0.0"}, diff --git a/elvis.config b/elvis.config index ef695458c..30178622a 100644 --- a/elvis.config +++ b/elvis.config @@ -1,4 +1,4 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- [ { elvis, diff --git a/rebar.config b/rebar.config index 7c5edc680..7e2eb7e54 100644 --- a/rebar.config +++ b/rebar.config @@ -1,4 +1,4 @@ -%% -*- mode:erlang -*- +%% -*- mode: erlang -*- %% This config file is the very basic config to compile emqx %% This allows emqx to be used as a dependency for other applications %% such as emqx module/plugin develpments and tests. From 43ce7276323ccd0e3c7705bee0a12bd5597765d7 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 15 Sep 2021 11:11:35 +0800 Subject: [PATCH 46/84] chore(gen_id): using emqx_misc:gen_id/0, /1 --- apps/emqx/test/emqx_misc_SUITE.erl | 4 ++ .../emqx_authz/src/emqx_authz_api_sources.erl | 6 +- .../src/emqx_connector_mqtt.erl | 2 +- .../src/emqx_plugin_libs_id.erl | 57 ------------------- .../test/emqx_plugin_libs_id_SUITE.erl | 28 --------- .../emqx_rule_engine/src/emqx_rule_engine.erl | 2 +- .../src/emqx_rule_sqltester.erl | 4 +- 7 files changed, 11 insertions(+), 92 deletions(-) delete mode 100644 apps/emqx_plugin_libs/src/emqx_plugin_libs_id.erl delete mode 100644 apps/emqx_plugin_libs/test/emqx_plugin_libs_id_SUITE.erl diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl index c3580545a..2ab5ee3e0 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx/test/emqx_misc_SUITE.erl @@ -147,3 +147,7 @@ t_now_to_secs(_) -> t_now_to_ms(_) -> ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))). +t_gen_id(_) -> + ?assertEqual(10, length(emqx_misc:gen_id(10))), + ?assertEqual(20, length(emqx_misc:gen_id(20))). + diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index a94a0e6c4..1586b1c88 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -450,21 +450,21 @@ write_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> CertPath = filename:join([emqx:get_config([node, data_dir]), "certs"]), CaCert = case maps:is_key(<<"cacertfile">>, SSL) of true -> - {ok, CaCertFile} = write_file(filename:join([CertPath, "cacert-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + {ok, CaCertFile} = write_file(filename:join([CertPath, "cacert-" ++ emqx_misc:gen_id() ++".pem"]), maps:get(<<"cacertfile">>, SSL)), CaCertFile; false -> "" end, Cert = case maps:is_key(<<"certfile">>, SSL) of true -> - {ok, CertFile} = write_file(filename:join([CertPath, "cert-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + {ok, CertFile} = write_file(filename:join([CertPath, "cert-" ++ emqx_misc:gen_id() ++".pem"]), maps:get(<<"certfile">>, SSL)), CertFile; false -> "" end, Key = case maps:is_key(<<"keyfile">>, SSL) of true -> - {ok, KeyFile} = write_file(filename:join([CertPath, "key-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + {ok, KeyFile} = write_file(filename:join([CertPath, "key-" ++ emqx_misc:gen_id() ++".pem"]), maps:get(<<"keyfile">>, SSL)), KeyFile; false -> "" diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 914d26d94..125b5cbe4 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -214,7 +214,7 @@ bridge_name(Prefix, Id) -> list_to_atom(str(Prefix) ++ ":" ++ str(Id)). clientid(Id) -> - list_to_binary(str(Id) ++ ":" ++ emqx_plugin_libs_id:gen(16)). + list_to_binary(str(Id) ++ ":" ++ emqx_misc:gen_id(16)). str(A) when is_atom(A) -> atom_to_list(A); diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_id.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_id.erl deleted file mode 100644 index ef5fb08c3..000000000 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_id.erl +++ /dev/null @@ -1,57 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_plugin_libs_id). - --export([gen/0, gen/1]). - --define(SHORT, 8). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- --spec(gen() -> list()). -gen() -> - gen(?SHORT). - --spec(gen(integer()) -> list()). -gen(Len) -> - BitLen = Len * 4, - <> = crypto:strong_rand_bytes(Len div 2), - int_to_hex(R, Len). - -%%------------------------------------------------------------------------------ -%% Internal Functions -%%------------------------------------------------------------------------------ - -int_to_hex(I, N) when is_integer(I), I >= 0 -> - int_to_hex([], I, 1, N). - -int_to_hex(L, I, Count, N) - when I < 16 -> - pad([int_to_hex(I) | L], N - Count); -int_to_hex(L, I, Count, N) -> - int_to_hex([int_to_hex(I rem 16) | L], I div 16, Count + 1, N). - -int_to_hex(I) when 0 =< I, I =< 9 -> - I + $0; -int_to_hex(I) when 10 =< I, I =< 15 -> - (I - 10) + $a. - -pad(L, 0) -> - L; -pad(L, Count) -> - pad([$0 | L], Count - 1). diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_id_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_id_SUITE.erl deleted file mode 100644 index d13144ed3..000000000 --- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_id_SUITE.erl +++ /dev/null @@ -1,28 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_plugin_libs_id_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_ct:all(?MODULE). - -t_gen(_) -> - ?assertEqual(10, length(emqx_plugin_libs_id:gen(10))), - ?assertEqual(20, length(emqx_plugin_libs_id:gen(20))). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 24cfecb03..bf0eb06e8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -507,7 +507,7 @@ rule_id() -> gen_id("rule:", fun emqx_rule_registry:get_rule/1). gen_id(Prefix, TestFun) -> - Id = iolist_to_binary([Prefix, emqx_plugin_libs_id:gen()]), + Id = iolist_to_binary([Prefix, emqx_misc:gen_id()]), case TestFun(Id) of not_found -> Id; _Res -> gen_id(Prefix, TestFun) diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 3076f414c..2f1edbeb2 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -48,8 +48,8 @@ test(#{<<"rawsql">> := Sql, <<"ctx">> := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["test_rule", emqx_plugin_libs_id:gen()]), - ActInstId = iolist_to_binary(["test_action", emqx_plugin_libs_id:gen()]), + RuleId = iolist_to_binary(["test_rule", emqx_misc:gen_id()]), + ActInstId = iolist_to_binary(["test_action", emqx_misc:gen_id()]), ok = emqx_rule_metrics:create_rule_metrics(RuleId), ok = emqx_rule_metrics:create_metrics(ActInstId), Rule = #rule{ From 9df842eeb87e73fd19405fe090806c3def67254d Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 16 Sep 2021 13:55:22 +0800 Subject: [PATCH 47/84] fix(test): fix test case of authz --- apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 3862e06d0..a0b65221c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -148,8 +148,8 @@ set_special_configs(_App) -> ok. init_per_testcase(t_api, Config) -> - meck:new(emqx_plugin_libs_id, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_plugin_libs_id, gen, fun() -> "fake" end), + meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_misc, gen_id, fun() -> "fake" end), meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx, get_config, fun([node, data_dir]) -> @@ -162,7 +162,7 @@ init_per_testcase(t_api, Config) -> init_per_testcase(_, Config) -> Config. end_per_testcase(t_api, _Config) -> - meck:unload(emqx_plugin_libs_id), + meck:unload(emqx_misc), meck:unload(emqx), ok; end_per_testcase(_, _Config) -> ok. From ddfc010fdb434778f9ced59f6579d39094016b74 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 16 Sep 2021 16:53:43 +0800 Subject: [PATCH 48/84] fix(resource): fix undefined function --- apps/emqx_resource/src/emqx_resource.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 02b49c47f..fdbb284ef 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -172,7 +172,7 @@ create_dry_run(ResourceType, Config) -> -spec create_dry_run_local(resource_type(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run_local(ResourceType, Config) -> - InstId = emqx_plugin_libs_id:gen(16), + InstId = emqx_misc:gen_id(16), call_instance(InstId, {create_dry_run, InstId, ResourceType, Config}). -spec recreate(instance_id(), resource_type(), resource_config(), term()) -> From 16c652586b797a83847f15224f49f3d1776e059a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 16 Sep 2021 23:16:31 +0200 Subject: [PATCH 49/84] fix(deps): pin typerefl verison from root rebar.configs --- apps/emqx/rebar.config | 1 + rebar.config | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 119d521fc..7a43249a4 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -10,6 +10,7 @@ %% `git_subdir` dependency in other projects. {deps, [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.4"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.3"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} diff --git a/rebar.config b/rebar.config index 7e2eb7e54..1e31c68e4 100644 --- a/rebar.config +++ b/rebar.config @@ -44,6 +44,7 @@ {deps, [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.4"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.9"}}} , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} From 13a03d8c1c8bce0e2e84662886774c8712ffebda Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 15 Sep 2021 20:57:38 +0200 Subject: [PATCH 50/84] feat(ssl): make possible to disable client-initiated ssl renegotiation --- apps/emqx/etc/emqx.conf | 10 +++++++++ apps/emqx/src/emqx_schema.erl | 38 +++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 6834d2a6e..42d9305c8 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1325,6 +1325,16 @@ example_common_ssl_options { ## Default: true ssl.secure_renegotiate = true + ## In protocols that support client-initiated renegotiation, + ## the cost of resources of such an operation is higher for the server than the client. + ## This can act as a vector for denial of service attacks. + ## The SSL application already takes measures to counter-act such attempts, + ## but client-initiated renegotiation can be strictly disabled by setting this option to false. + ## The default value is true. Note that disabling renegotiation can result in + ## long-lived connections becoming unusable due to limits on + ## the number of messages the underlying cipher suite can encipher. + ssl.client_renegotiation = true + ## An important security setting, it forces the cipher to be set based ## on the server-specified order instead of the client-specified order, ## hence enforcing the (usually more properly configured) security diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 7ff517c31..d7caeb971 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -904,9 +904,10 @@ filter(Opts) -> ssl(Defaults) -> D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, + Df = fun (Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end, [ {"enable", sc(boolean(), - #{ default => D("enable") + #{ default => Df("enable", false) }) } , {"cacertfile", @@ -926,37 +927,58 @@ ssl(Defaults) -> } , {"verify", sc(hoconsc:union([verify_peer, verify_none]), - #{ default => D("verify") + #{ default => Df("verify", verify_none) }) } , {"fail_if_no_peer_cert", sc(boolean(), - #{ default => D("fail_if_no_peer_cert") + #{ default => Df("fail_if_no_peer_cert", false) }) } , {"secure_renegotiate", sc(boolean(), - #{ default => D("secure_renegotiate") + #{ default => Df("secure_renegotiate", true) + , desc => """ +SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +you drop support for the insecure renegotiation, prone to MitM attacks. +""" + }) + } + , {"client_renegotiation", + sc(boolean(), + #{ default => Df("client_renegotiation", true) + , desc => """ +In protocols that support client-initiated renegotiation, +the cost of resources of such an operation is higher for the server than the client. +This can act as a vector for denial of service attacks. +The SSL application already takes measures to counter-act such attempts, +but client-initiated renegotiation can be strictly disabled by setting this option to false. +The default value is true. Note that disabling renegotiation can result in +long-lived connections becoming unusable due to limits on +the number of messages the underlying cipher suite can encipher. +""" }) } , {"reuse_sessions", sc(boolean(), - #{ default => D("reuse_sessions") + #{ default => Df("reuse_sessions", true) }) } , {"honor_cipher_order", sc(boolean(), - #{ default => D("honor_cipher_order") + #{ default => Df("honor_cipher_order", true) }) } , {"handshake_timeout", sc(duration(), - #{ default => D("handshake_timeout") + #{ default => Df("handshake_timeout", "15s") }) } , {"depth", sc(integer(), - #{default => D("depth") + #{default => Df("depth", 10) }) } , {"password", From 3dcccc0b332a30c6347f44d2a61d62ddcfb3c27a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 16 Sep 2021 09:32:17 +0200 Subject: [PATCH 51/84] refactor(authn): call lists:foreach instaed of list comprehension --- apps/emqx_authn/include/emqx_authn.hrl | 5 +++++ apps/emqx_authn/src/emqx_authn_app.erl | 13 ++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 5eef08012..1c5c00e85 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -14,6 +14,9 @@ %% limitations under the License. %%-------------------------------------------------------------------- +-ifndef(EMQX_AUTHN_HRL). +-define(EMQX_AUTHN_HRL, true). + -define(APP, emqx_authn). -define(AUTHN, emqx_authentication). @@ -23,3 +26,5 @@ -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). -define(AUTH_SHARD, emqx_authn_shard). + +-endif. diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 4d4a22791..98d53e438 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -45,17 +45,20 @@ stop(_State) -> %%------------------------------------------------------------------------------ add_providers() -> - _ = [?AUTHN:add_provider(AuthNType, Provider) || {AuthNType, Provider} <- providers()], ok. + lists:foreach(fun(AuthNType, Provider}) -> + ?AUTHN:add_provider(AuthNType, Provider) + end, providers()). remove_providers() -> - _ = [?AUTHN:remove_provider(AuthNType) || {AuthNType, _} <- providers()], ok. + lists:foreach(fun({AuthNType, _}) -> + ?AUTHN:remove_provider(AuthNType) + end, providers()). initialize() -> ?AUTHN:initialize_authentication(?GLOBAL, emqx:get_raw_config([authentication], [])), lists:foreach(fun({ListenerID, ListenerConfig}) -> ?AUTHN:initialize_authentication(ListenerID, maps:get(authentication, ListenerConfig, [])) - end, emqx_listeners:list()), - ok. + end, emqx_listeners:list()). providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} @@ -66,4 +69,4 @@ providers() -> , {{'password-based', 'http-server'}, emqx_authn_http} , {jwt, emqx_authn_jwt} , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} - ]. \ No newline at end of file + ]. From 0b432a6a778cc141c364f366cc34b0764c4c1d7e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 16 Sep 2021 10:53:41 +0200 Subject: [PATCH 52/84] refactor(auth): rename functions from may_xx to maybe_xx --- apps/emqx/src/emqx_authentication.erl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 483305aa4..9b249c8a7 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -266,8 +266,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} move_authenticator(ChainName, AuthenticatorID, Position). check_config(Config) -> - #{authentication := CheckedConfig} = hocon_schema:check_plain(emqx_authentication, - #{<<"authentication">> => Config}, #{nullable => true, atom_key => true}), + #{authentication := CheckedConfig} = + hocon_schema:check_plain(?MODULE, #{<<"authentication">> => Config}, + #{nullable => true, atom_key => true}), CheckedConfig. %%------------------------------------------------------------------------------ @@ -476,7 +477,7 @@ handle_call({delete_chain, Name}, _From, State) -> [#chain{authenticators = Authenticators}] -> _ = [do_delete_authenticator(Authenticator) || Authenticator <- Authenticators], true = ets:delete(?CHAINS_TAB, Name), - reply(ok, may_unhook(State)) + reply(ok, maybe_unhook(State)) end; handle_call({lookup_chain, Name}, _From, State) -> @@ -506,7 +507,7 @@ handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Pro end end, Reply = update_chain(ChainName, UpdateFun), - reply(Reply, may_hook(State)); + reply(Reply, maybe_hook(State)); handle_call({delete_authenticator, ChainName, AuthenticatorID}, _From, State) -> UpdateFun = @@ -521,7 +522,7 @@ handle_call({delete_authenticator, ChainName, AuthenticatorID}, _From, State) -> end end, Reply = update_chain(ChainName, UpdateFun), - reply(Reply, may_unhook(State)); + reply(Reply, maybe_unhook(State)); handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, State) -> UpdateFun = @@ -726,30 +727,30 @@ global_chain(stomp) -> global_chain(_) -> 'unknown:global'. -may_hook(#{hooked := false} = State) -> +maybe_hook(#{hooked := false} = State) -> case lists:any(fun(#chain{authenticators = []}) -> false; (_) -> true end, ets:tab2list(?CHAINS_TAB)) of true -> - _ = emqx:hook('client.authenticate', {emqx_authentication, authenticate, []}), + _ = emqx:hook('client.authenticate', {?MODULE, authenticate, []}), State#{hooked => true}; false -> State end; -may_hook(State) -> +maybe_hook(State) -> State. -may_unhook(#{hooked := true} = State) -> +maybe_unhook(#{hooked := true} = State) -> case lists:all(fun(#chain{authenticators = []}) -> true; (_) -> false end, ets:tab2list(?CHAINS_TAB)) of true -> - _ = emqx:unhook('client.authenticate', {emqx_authentication, authenticate, []}), + _ = emqx:unhook('client.authenticate', {?MODULE, authenticate, []}), State#{hooked => false}; false -> State end; -may_unhook(State) -> +maybe_unhook(State) -> State. do_create_authenticator(ChainName, AuthenticatorID, #{enable := Enable} = Config, Providers) -> @@ -773,7 +774,7 @@ do_create_authenticator(ChainName, AuthenticatorID, #{enable := Enable} = Config do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> _ = Provider:destroy(State), ok. - + replace_authenticator(ID, Authenticator, Authenticators) -> lists:keyreplace(ID, #authenticator.id, Authenticators, Authenticator). From 0877fb5569f82424eb5b9dc878de933964899b11 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 16 Sep 2021 22:58:42 +0200 Subject: [PATCH 53/84] refactor(authn): register providers in batch --- apps/emqx/src/emqx_authentication.erl | 49 ++++++++--- apps/emqx/test/emqx_authentication_SUITE.erl | 88 ++++++++++++-------- apps/emqx_authn/src/emqx_authn_app.erl | 17 ++-- 3 files changed, 93 insertions(+), 61 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 9b249c8a7..7df2036ac 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -40,8 +40,10 @@ , stop/0 ]). --export([ add_provider/2 - , remove_provider/1 +-export([ register_provider/2 + , register_providers/1 + , deregister_provider/1 + , deregister_providers/1 , create_chain/1 , delete_chain/1 , lookup_chain/1 @@ -334,13 +336,27 @@ stop() -> get_refs() -> gen_server:call(?MODULE, get_refs). --spec add_provider(authn_type(), module()) -> ok. -add_provider(AuthNType, Provider) -> - gen_server:call(?MODULE, {add_provider, AuthNType, Provider}). +%% @doc Register authentication providers. +%% A provider is a tuple of `AuthNType' the module which implements +%% the authenticator callbacks. +%% For example, ``[{{'password-based', redis}, emqx_authn_redis}]'' +%% NOTE: Later registered provider may override earlier registered if they +%% happen to clash the same `AuthNType'. +-spec register_providers([{authn_type(), module()}]) -> ok. +register_providers(Providers) -> + gen_server:call(?MODULE, {register_providers, Providers}). --spec remove_provider(authn_type()) -> ok. -remove_provider(AuthNType) -> - gen_server:call(?MODULE, {remove_provider, AuthNType}). +-spec register_provider(authn_type(), module()) -> ok. +register_provider(AuthNType, Provider) -> + register_providers([{AuthNType, Provider}]). + +-spec deregister_providers([authn_type()]) -> ok. +deregister_providers(AuthNTypes) when is_list(AuthNTypes) -> + gen_server:call(?MODULE, {deregister_providers, AuthNTypes}). + +-spec deregister_provider(authn_type()) -> ok. +deregister_provider(AuthNType) -> + deregister_providers([AuthNType]). -spec create_chain(chain_name()) -> {ok, chain()} | {error, term()}. create_chain(Name) -> @@ -447,11 +463,20 @@ init(_Opts) -> ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), {ok, #{hooked => false, providers => #{}}}. -handle_call({add_provider, AuthNType, Provider}, _From, #{providers := Providers} = State) -> - reply(ok, State#{providers := Providers#{AuthNType => Provider}}); +handle_call({register_providers, Providers}, _From, + #{providers := Reg0} = State) -> + case lists:filter(fun({T, _}) -> maps:is_key(T, Reg0) end, Providers) of + [] -> + Reg = lists:foldl(fun({AuthNType, Module}, Pin) -> + Pin#{AuthNType => Module} + end, Reg0, Providers), + reply(ok, State#{providers := Reg}); + Clashes -> + reply({error, {authentication_type_clash, Clashes}}, State) + end; -handle_call({remove_provider, AuthNType}, _From, #{providers := Providers} = State) -> - reply(ok, State#{providers := maps:remove(AuthNType, Providers)}); +handle_call({deregister_providers, AuthNTypes}, _From, #{providers := Providers} = State) -> + reply(ok, State#{providers := maps:without(AuthNTypes, Providers)}); handle_call(get_refs, _From, #{providers := Providers} = State) -> Refs = lists:foldl(fun({_, Provider}, Acc) -> diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index ff219e64c..15aecc269 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -36,6 +36,7 @@ ]). -define(AUTHN, emqx_authentication). +-define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -92,20 +93,22 @@ end_per_suite(_) -> emqx_ct_helpers:stop_apps([]), ok. -init_per_testcase(_, Config) -> +init_per_testcase(Case, Config) -> meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx, get_config, fun([node, data_dir]) -> {data_dir, Data} = lists:keyfind(data_dir, 1, Config), Data; (C) -> meck:passthrough([C]) end), - Config. + ?MODULE:Case({'init', Config}). -end_per_testcase(_, _Config) -> +end_per_testcase(Case, Config) -> + _ = ?MODULE:Case({'end', Config}), meck:unload(emqx), ok. -t_chain(_) -> +t_chain({_, Config}) -> Config; +t_chain(Config) when is_list(Config) -> % CRUD of authentication chain ChainName = 'test', ?assertMatch({ok, []}, ?AUTHN:list_chains()), @@ -117,7 +120,10 @@ t_chain(_) -> ?assertMatch({error, {not_found, {chain, ChainName}}}, ?AUTHN:lookup_chain(ChainName)), ok. -t_authenticator(_) -> +t_authenticator({'init', Config}) -> + [{"auth1", {'password-based', 'built-in-database'}}, + {"auth2", {'password-based', mysql}} | Config]; +t_authenticator(Config) when is_list(Config) -> ChainName = 'test', AuthenticatorConfig1 = #{mechanism => 'password-based', backend => 'built-in-database', @@ -129,8 +135,8 @@ t_authenticator(_) -> % Create an authenticator when the provider does not exist ?assertEqual({error, no_available_provider}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), - AuthNType1 = {'password-based', 'built-in-database'}, - ?AUTHN:add_provider(AuthNType1, ?MODULE), + AuthNType1 = ?config("auth1"), + register_provider(AuthNType1, ?MODULE), ID1 = <<"password-based:built-in-database">>, % CRUD of authencaticator @@ -144,8 +150,8 @@ t_authenticator(_) -> ?assertMatch({ok, []}, ?AUTHN:list_authenticators(ChainName)), % Multiple authenticators exist at the same time - AuthNType2 = {'password-based', mysql}, - ?AUTHN:add_provider(AuthNType2, ?MODULE), + AuthNType2 = ?config("auth2"), + register_provider(AuthNType2, ?MODULE), ID2 = <<"password-based:mysql">>, AuthenticatorConfig2 = #{mechanism => 'password-based', backend => mysql, @@ -160,15 +166,18 @@ t_authenticator(_) -> ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, bottom)), ?assertMatch({ok, [#{id := ID1}, #{id := ID2}]}, ?AUTHN:list_authenticators(ChainName)), ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, {before, ID1})), - ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), - - ?AUTHN:delete_chain(ChainName), - ?AUTHN:remove_provider(AuthNType1), - ?AUTHN:remove_provider(AuthNType2), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)); +t_authenticator({'end', Config}) -> + ?AUTHN:delete_chain(test), + ?AUTHN:deregister_providers([?config("auth1"), ?config("auth2")]), ok. -t_authenticate(_) -> - ListenerID = 'tcp:default', +t_authenticate({init, Config}) -> + [{listener_id, 'tcp:default'}, + {authn_type, {'password-based', 'built-in-database'}} | Config]; +t_authenticate(Config) when is_list(Config) -> + ListenerID = ?config(listener_id), + AuthNType = ?config(authn_type), ClientInfo = #{zone => default, listener => ListenerID, protocol => mqtt, @@ -176,8 +185,7 @@ t_authenticate(_) -> password => <<"any">>}, ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), - AuthNType = {'password-based', 'built-in-database'}, - ?AUTHN:add_provider(AuthNType, ?MODULE), + register_provider(AuthNType, ?MODULE), AuthenticatorConfig = #{mechanism => 'password-based', backend => 'built-in-database', @@ -185,21 +193,24 @@ t_authenticate(_) -> ?AUTHN:create_chain(ListenerID), ?assertMatch({ok, _}, ?AUTHN:create_authenticator(ListenerID, AuthenticatorConfig)), ?assertEqual({ok, #{is_superuser => true}}, emqx_access_control:authenticate(ClientInfo)), - ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo#{username => <<"bad">>})), - - ?AUTHN:delete_chain(ListenerID), - ?AUTHN:remove_provider(AuthNType), + ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo#{username => <<"bad">>})); +t_authenticate({'end', Config}) -> + ?AUTHN:delete_chain(?config(listener_id)), + ?AUTHN:deregister_provider(?config(authn_type)), ok. -t_update_config(_) -> - emqx_config_handler:add_handler([authentication], emqx_authentication), - +t_update_config({init, Config}) -> + Global = 'mqtt:global', AuthNType1 = {'password-based', 'built-in-database'}, AuthNType2 = {'password-based', mysql}, - ?AUTHN:add_provider(AuthNType1, ?MODULE), - ?AUTHN:add_provider(AuthNType2, ?MODULE), - - Global = 'mqtt:global', + [{global, Global}, + {"auth1", AuthNType1}, + {"auth2", AuthNType2} | Config]; +t_update_config(Config) when is_list(Config) -> + emqx_config_handler:add_handler([authentication], emqx_authentication), + ok = register_provider(?config("auth1"), ?MODULE), + ok = register_provider(?config("auth2"), ?MODULE), + Global = ?config(global), AuthenticatorConfig1 = #{mechanism => 'password-based', backend => 'built-in-database', enable => true}, @@ -208,7 +219,7 @@ t_update_config(_) -> enable => true}, ID1 = <<"password-based:built-in-database">>, ID2 = <<"password-based:mysql">>, - + ?assertMatch({ok, []}, ?AUTHN:list_chains()), ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), @@ -240,14 +251,14 @@ t_update_config(_) -> ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ListenerID)), ?assertMatch({ok, _}, update_config(ConfKeyPath, {delete_authenticator, ListenerID, ID1})), - ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), - - ?AUTHN:delete_chain(Global), - ?AUTHN:remove_provider(AuthNType1), - ?AUTHN:remove_provider(AuthNType2), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)); +t_update_config({'end', Config}) -> + ?AUTHN:delete_chain(?config(global)), + ?AUTHN:deregister_providers([?config("auth1"), ?config("auth2")]), ok. -t_convert_cert_options(_) -> +t_convert_cert_options({_, Config}) -> Config; +t_convert_cert_options(Config) when is_list(Config) -> Certs = certs([ {<<"keyfile">>, "key.pem"} , {<<"certfile">>, "cert.pem"} , {<<"cacertfile">>, "cacert.pem"} @@ -284,4 +295,7 @@ certs(Certs) -> diff_cert(CertFile, CertPem2) -> {ok, CertPem1} = file:read_file(CertFile), - ?AUTHN:diff_cert(CertPem1, CertPem2). \ No newline at end of file + ?AUTHN:diff_cert(CertPem1, CertPem2). + +register_provider(Type, Module) -> + ok = ?AUTHN:register_providers([{Type, Module}]). diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 98d53e438..d297c9042 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -32,34 +32,27 @@ start(_StartType, _StartArgs) -> ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), - ok = add_providers(), + ok = ?AUTHN:register_providers(providers()), ok = initialize(), {ok, Sup}. stop(_State) -> - ok = remove_providers(), + ok = ?AUTHN:deregister_providers(provider_types()), ok. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ -add_providers() -> - lists:foreach(fun(AuthNType, Provider}) -> - ?AUTHN:add_provider(AuthNType, Provider) - end, providers()). - -remove_providers() -> - lists:foreach(fun({AuthNType, _}) -> - ?AUTHN:remove_provider(AuthNType) - end, providers()). - initialize() -> ?AUTHN:initialize_authentication(?GLOBAL, emqx:get_raw_config([authentication], [])), lists:foreach(fun({ListenerID, ListenerConfig}) -> ?AUTHN:initialize_authentication(ListenerID, maps:get(authentication, ListenerConfig, [])) end, emqx_listeners:list()). +provider_types() -> + lists:map(fun({Type, _Module}) -> Type end, providers()). + providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} From 6cf1107f42189204db237cbd39237d5817ad343c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 16 Sep 2021 23:01:56 +0200 Subject: [PATCH 54/84] refactor(authn): ensure infinity timeout for gen_server call --- apps/emqx/src/emqx_authentication.erl | 34 ++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 7df2036ac..f047a1e41 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -334,7 +334,7 @@ stop() -> -spec get_refs() -> {ok, Refs} when Refs :: [{authn_type(), module()}]. get_refs() -> - gen_server:call(?MODULE, get_refs). + call(get_refs). %% @doc Register authentication providers. %% A provider is a tuple of `AuthNType' the module which implements @@ -344,7 +344,7 @@ get_refs() -> %% happen to clash the same `AuthNType'. -spec register_providers([{authn_type(), module()}]) -> ok. register_providers(Providers) -> - gen_server:call(?MODULE, {register_providers, Providers}). + call({register_providers, Providers}). -spec register_provider(authn_type(), module()) -> ok. register_provider(AuthNType, Provider) -> @@ -352,7 +352,7 @@ register_provider(AuthNType, Provider) -> -spec deregister_providers([authn_type()]) -> ok. deregister_providers(AuthNTypes) when is_list(AuthNTypes) -> - gen_server:call(?MODULE, {deregister_providers, AuthNTypes}). + call({deregister_providers, AuthNTypes}). -spec deregister_provider(authn_type()) -> ok. deregister_provider(AuthNType) -> @@ -360,15 +360,15 @@ deregister_provider(AuthNType) -> -spec create_chain(chain_name()) -> {ok, chain()} | {error, term()}. create_chain(Name) -> - gen_server:call(?MODULE, {create_chain, Name}). + call({create_chain, Name}). -spec delete_chain(chain_name()) -> ok | {error, term()}. delete_chain(Name) -> - gen_server:call(?MODULE, {delete_chain, Name}). + call({delete_chain, Name}). -spec lookup_chain(chain_name()) -> {ok, chain()} | {error, term()}. lookup_chain(Name) -> - gen_server:call(?MODULE, {lookup_chain, Name}). + call({lookup_chain, Name}). -spec list_chains() -> {ok, [chain()]}. list_chains() -> @@ -377,15 +377,15 @@ list_chains() -> -spec create_authenticator(chain_name(), config()) -> {ok, authenticator()} | {error, term()}. create_authenticator(ChainName, Config) -> - gen_server:call(?MODULE, {create_authenticator, ChainName, Config}). + call({create_authenticator, ChainName, Config}). -spec delete_authenticator(chain_name(), authenticator_id()) -> ok | {error, term()}. delete_authenticator(ChainName, AuthenticatorID) -> - gen_server:call(?MODULE, {delete_authenticator, ChainName, AuthenticatorID}). + call({delete_authenticator, ChainName, AuthenticatorID}). -spec update_authenticator(chain_name(), authenticator_id(), config()) -> {ok, authenticator()} | {error, term()}. update_authenticator(ChainName, AuthenticatorID, Config) -> - gen_server:call(?MODULE, {update_authenticator, ChainName, AuthenticatorID, Config}). + call({update_authenticator, ChainName, AuthenticatorID, Config}). -spec lookup_authenticator(chain_name(), authenticator_id()) -> {ok, authenticator()} | {error, term()}. lookup_authenticator(ChainName, AuthenticatorID) -> @@ -412,32 +412,32 @@ list_authenticators(ChainName) -> -spec move_authenticator(chain_name(), authenticator_id(), position()) -> ok | {error, term()}. move_authenticator(ChainName, AuthenticatorID, Position) -> - gen_server:call(?MODULE, {move_authenticator, ChainName, AuthenticatorID, Position}). + call({move_authenticator, ChainName, AuthenticatorID, Position}). -spec import_users(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. import_users(ChainName, AuthenticatorID, Filename) -> - gen_server:call(?MODULE, {import_users, ChainName, AuthenticatorID, Filename}). + call({import_users, ChainName, AuthenticatorID, Filename}). -spec add_user(chain_name(), authenticator_id(), user_info()) -> {ok, user_info()} | {error, term()}. add_user(ChainName, AuthenticatorID, UserInfo) -> - gen_server:call(?MODULE, {add_user, ChainName, AuthenticatorID, UserInfo}). + call({add_user, ChainName, AuthenticatorID, UserInfo}). -spec delete_user(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. delete_user(ChainName, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {delete_user, ChainName, AuthenticatorID, UserID}). + call({delete_user, ChainName, AuthenticatorID, UserID}). -spec update_user(chain_name(), authenticator_id(), binary(), map()) -> {ok, user_info()} | {error, term()}. update_user(ChainName, AuthenticatorID, UserID, NewUserInfo) -> - gen_server:call(?MODULE, {update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}). + call({update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}). -spec lookup_user(chain_name(), authenticator_id(), binary()) -> {ok, user_info()} | {error, term()}. lookup_user(ChainName, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {lookup_user, ChainName, AuthenticatorID, UserID}). + call({lookup_user, ChainName, AuthenticatorID, UserID}). %% TODO: Support pagination -spec list_users(chain_name(), authenticator_id()) -> {ok, [user_info()]} | {error, term()}. list_users(ChainName, AuthenticatorID) -> - gen_server:call(?MODULE, {list_users, ChainName, AuthenticatorID}). + call({list_users, ChainName, AuthenticatorID}). -spec generate_id(config()) -> authenticator_id(). generate_id(#{mechanism := Mechanism0, backend := Backend0}) -> @@ -901,3 +901,5 @@ to_list(L) when is_list(L) -> to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L). + +call(Call) -> gen_server:call(?MODULE, Call, infinity). From fa467d0741b35fd4c0994b2f46b2d02e98fe23ae Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 16 Sep 2021 18:11:26 +0800 Subject: [PATCH 55/84] chore(authz): rename pgsql to postgresql Signed-off-by: zhanghongtong --- apps/emqx_authz/README.md | 4 ++-- apps/emqx_authz/etc/emqx_authz.conf | 2 +- apps/emqx_authz/src/emqx_authz.erl | 6 +++-- apps/emqx_authz/src/emqx_authz_api_schema.erl | 8 +++---- ...hz_pgsql.erl => emqx_authz_postgresql.erl} | 6 ++--- apps/emqx_authz/src/emqx_authz_schema.erl | 13 ++++++----- apps/emqx_authz/test/emqx_authz_SUITE.erl | 22 +++++++++---------- .../test/emqx_authz_api_sources_SUITE.erl | 18 +++++++-------- ...TE.erl => emqx_authz_postgresql_SUITE.erl} | 4 ++-- 9 files changed, 44 insertions(+), 39 deletions(-) rename apps/emqx_authz/src/{emqx_authz_pgsql.erl => emqx_authz_postgresql.erl} (96%) rename apps/emqx_authz/test/{emqx_authz_pgsql_SUITE.erl => emqx_authz_postgresql_SUITE.erl} (98%) diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md index bc94578c0..a44297a55 100644 --- a/apps/emqx_authz/README.md +++ b/apps/emqx_authz/README.md @@ -26,7 +26,7 @@ authz:{ sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" }, { - type: pgsql + type: postgresql config: { server: "127.0.0.1:5432" database: mqtt @@ -96,7 +96,7 @@ Sample data in the default configuration: INSERT INTO mqtt_authz (ipaddress, username, clientid, action, permission, topic) VALUES ('127.0.0.1', '', '', 'subscribe', 'allow', '$SYS/#'); ``` -#### Pgsql +#### PostgreSQL Create Example Table diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index c2856f0b5..19bb2737b 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -25,7 +25,7 @@ authorization { # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" # }, # { - # type: pgsql + # type: postgresql # server: "127.0.0.1:5432" # database: mqtt # pool_size: 1 diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 6c7ceb6c2..b4debf3fb 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -39,7 +39,7 @@ -export([post_config_update/4, pre_config_update/2]). -define(CONF_KEY_PATH, [authorization, sources]). --define(SOURCE_TYPES, [file, http, mongo, mysql, pgsql, redis]). +-define(SOURCE_TYPES, [file, http, mongo, mysql, postgresql, redis]). -spec(register_metrics() -> ok). register_metrics() -> @@ -309,7 +309,7 @@ init_source(#{enable := true, type := DB, query := SQL } = Source) when DB =:= mysql; - DB =:= pgsql -> + DB =:= postgresql -> Mod = authz_module(DB), case create_resource(Source) of {error, Reason} -> error({load_config_error, Reason}); @@ -407,6 +407,8 @@ create_resource(#{type := DB} = Source) -> authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). +connector_module(postgresql) -> + emqx_connector_pgsql; connector_module(Type) -> list_to_existing_atom("emqx_connector_" ++ atom_to_list(Type)). diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 158e26eab..4fd2f8ca1 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -46,7 +46,7 @@ definitions() -> , minirest:ref(<<"mongo_rs">>) , minirest:ref(<<"mongo_sharded">>) , minirest:ref(<<"mysql">>) - , minirest:ref(<<"pgsql">>) + , minirest:ref(<<"postgresql">>) , minirest:ref(<<"redis_single">>) , minirest:ref(<<"redis_sentinel">>) , minirest:ref(<<"redis_cluster">>) @@ -335,8 +335,8 @@ definitions() -> properties => #{ type => #{ type => string, - enum => [<<"pgsql">>], - example => <<"pgsql">> + enum => [<<"postgresql">>], + example => <<"postgresql">> }, enable => #{ type => boolean, @@ -501,7 +501,7 @@ definitions() -> , #{<<"mongo_rs">> => MongoRs} , #{<<"mongo_sharded">> => MongoSharded} , #{<<"mysql">> => Mysql} - , #{<<"pgsql">> => Pgsql} + , #{<<"postgresql">> => Pgsql} , #{<<"redis_single">> => RedisSingle} , #{<<"redis_sentinel">> => RedisSentinel} , #{<<"redis_cluster">> => RedisCluster} diff --git a/apps/emqx_authz/src/emqx_authz_pgsql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl similarity index 96% rename from apps/emqx_authz/src/emqx_authz_pgsql.erl rename to apps/emqx_authz/src/emqx_authz_postgresql.erl index 3e1f40fb2..ead2d985d 100644 --- a/apps/emqx_authz/src/emqx_authz_pgsql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authz_pgsql). +-module(emqx_authz_postgresql). -include("emqx_authz.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -32,7 +32,7 @@ -endif. description() -> - "AuthZ with pgsql". + "AuthZ with postgresql". parse_query(undefined) -> undefined; @@ -59,7 +59,7 @@ authorize(Client, PubSub, Topic, {ok, Columns, Rows} -> do_authorize(Client, PubSub, Topic, Columns, Rows); {error, Reason} -> - ?LOG(error, "[AuthZ] Query pgsql error: ~p~n", [Reason]), + ?LOG(error, "[AuthZ] Query postgresql error: ~p~n", [Reason]), nomatch end. diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index e17a55d0a..bda8d8e74 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -33,7 +33,7 @@ fields("authorization") -> , hoconsc:ref(?MODULE, mongo_rs) , hoconsc:ref(?MODULE, mongo_sharded) , hoconsc:ref(?MODULE, mysql) - , hoconsc:ref(?MODULE, pgsql) + , hoconsc:ref(?MODULE, postgresql) , hoconsc:ref(?MODULE, redis_single) , hoconsc:ref(?MODULE, redis_sentinel) , hoconsc:ref(?MODULE, redis_cluster) @@ -131,9 +131,12 @@ fields(mongo_sharded) -> fields(mysql) -> connector_fields(mysql) ++ [ {query, query()} ]; -fields(pgsql) -> - connector_fields(pgsql) ++ - [ {query, query()} ]; +fields(postgresql) -> + [ {type, #{type => postgresql}} + , {enable, #{type => boolean(), + default => true}} + , {query, query()} + ] ++ emqx_connector_pgsql:fields(config); fields(redis_single) -> connector_fields(redis, single) ++ [ {cmd, query()} ]; @@ -181,4 +184,4 @@ connector_fields(DB, Fields) -> to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> - binary_to_list(B). \ No newline at end of file + binary_to_list(B). diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index fe1f04bd2..bed20e0e4 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -88,7 +88,7 @@ init_per_testcase(_, Config) -> <<"ssl">> => #{<<"enable">> => false}, <<"query">> => <<"abcb">> }). --define(SOURCE4, #{<<"type">> => <<"pgsql">>, +-define(SOURCE4, #{<<"type">> => <<"postgresql">>, <<"enable">> => true, <<"server">> => <<"127.0.0.1:27017">>, <<"pool_size">> => 1, @@ -130,7 +130,7 @@ t_update_source(_) -> ?assertMatch([ #{type := http, enable := true} , #{type := mongo, enable := true} , #{type := mysql, enable := true} - , #{type := pgsql, enable := true} + , #{type := postgresql, enable := true} , #{type := redis, enable := true} , #{type := file, enable := true} ], emqx:get_config([authorization, sources], [])), @@ -138,14 +138,14 @@ t_update_source(_) -> {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, mongo}, ?SOURCE2#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, pgsql}, ?SOURCE4#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, postgresql}, ?SOURCE4#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}), ?assertMatch([ #{type := http, enable := false} , #{type := mongo, enable := false} , #{type := mysql, enable := false} - , #{type := pgsql, enable := false} + , #{type := postgresql, enable := false} , #{type := redis, enable := false} , #{type := file, enable := false} ], emqx:get_config([authorization, sources], [])), @@ -157,13 +157,13 @@ t_move_source(_) -> ?assertMatch([ #{type := http} , #{type := mongo} , #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := redis} , #{type := file} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(pgsql, <<"top">>), - ?assertMatch([ #{type := pgsql} + {ok, _} = emqx_authz:move(postgresql, <<"top">>), + ?assertMatch([ #{type := postgresql} , #{type := http} , #{type := mongo} , #{type := mysql} @@ -172,7 +172,7 @@ t_move_source(_) -> ], emqx_authz:lookup()), {ok, _} = emqx_authz:move(http, <<"bottom">>), - ?assertMatch([ #{type := pgsql} + ?assertMatch([ #{type := postgresql} , #{type := mongo} , #{type := mysql} , #{type := redis} @@ -180,9 +180,9 @@ t_move_source(_) -> , #{type := http} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(mysql, #{<<"before">> => pgsql}), + {ok, _} = emqx_authz:move(mysql, #{<<"before">> => postgresql}), ?assertMatch([ #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := mongo} , #{type := redis} , #{type := file} @@ -191,7 +191,7 @@ t_move_source(_) -> {ok, _} = emqx_authz:move(mongo, #{<<"after">> => http}), ?assertMatch([ #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := redis} , #{type := file} , #{type := http} diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 3862e06d0..678c6e894 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -67,7 +67,7 @@ <<"ssl">> => #{<<"enable">> => false}, <<"query">> => <<"abcb">> }). --define(SOURCE4, #{<<"type">> => <<"pgsql">>, +-define(SOURCE4, #{<<"type">> => <<"postgresql">>, <<"enable">> => true, <<"server">> => <<"127.0.0.1:5432">>, <<"pool_size">> => 1, @@ -183,7 +183,7 @@ t_api(_) -> ?assertMatch([ #{<<"type">> := <<"http">>} , #{<<"type">> := <<"mongo">>} , #{<<"type">> := <<"mysql">>} - , #{<<"type">> := <<"pgsql">>} + , #{<<"type">> := <<"postgresql">>} , #{<<"type">> := <<"redis">>} , #{<<"type">> := <<"file">>} ], Sources), @@ -227,13 +227,13 @@ t_move_source(_) -> ?assertMatch([ #{type := http} , #{type := mongo} , #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := redis} ], emqx_authz:lookup()), - {ok, 204, _} = request(post, uri(["authorization", "sources", "pgsql", "move"]), + {ok, 204, _} = request(post, uri(["authorization", "sources", "postgresql", "move"]), #{<<"position">> => <<"top">>}), - ?assertMatch([ #{type := pgsql} + ?assertMatch([ #{type := postgresql} , #{type := http} , #{type := mongo} , #{type := mysql} @@ -242,7 +242,7 @@ t_move_source(_) -> {ok, 204, _} = request(post, uri(["authorization", "sources", "http", "move"]), #{<<"position">> => <<"bottom">>}), - ?assertMatch([ #{type := pgsql} + ?assertMatch([ #{type := postgresql} , #{type := mongo} , #{type := mysql} , #{type := redis} @@ -250,9 +250,9 @@ t_move_source(_) -> ], emqx_authz:lookup()), {ok, 204, _} = request(post, uri(["authorization", "sources", "mysql", "move"]), - #{<<"position">> => #{<<"before">> => <<"pgsql">>}}), + #{<<"position">> => #{<<"before">> => <<"postgresql">>}}), ?assertMatch([ #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := mongo} , #{type := redis} , #{type := http} @@ -261,7 +261,7 @@ t_move_source(_) -> {ok, 204, _} = request(post, uri(["authorization", "sources", "mongo", "move"]), #{<<"position">> => #{<<"after">> => <<"http">>}}), ?assertMatch([ #{type := mysql} - , #{type := pgsql} + , #{type := postgresql} , #{type := redis} , #{type := http} , #{type := mongo} diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl similarity index 98% rename from apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl rename to apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 570ea0e77..9526adcda 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -13,7 +13,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authz_pgsql_SUITE). +-module(emqx_authz_postgresql_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -47,7 +47,7 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), - Rules = [#{<<"type">> => <<"pgsql">>, + Rules = [#{<<"type">> => <<"postgresql">>, <<"server">> => <<"127.0.0.1:27017">>, <<"pool_size">> => 1, <<"database">> => <<"mqtt">>, From 6ed0bdc66590b33c963dc7a77e374da49dd6ccf5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 17 Sep 2021 15:23:25 +0800 Subject: [PATCH 56/84] fix(cluster-call): MinDelay clumiscalculation, don't sleep in first check. --- apps/emqx_machine/src/emqx_cluster_rpc.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_machine/src/emqx_cluster_rpc.erl b/apps/emqx_machine/src/emqx_cluster_rpc.erl index c747f7654..d4a7bfee8 100644 --- a/apps/emqx_machine/src/emqx_cluster_rpc.erl +++ b/apps/emqx_machine/src/emqx_cluster_rpc.erl @@ -101,9 +101,9 @@ multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNu end end, End = erlang:monotonic_time(), - MinDelay = erlang:convert_time_unit(Begin - End, native, millisecond) + 50, + MinDelay = erlang:convert_time_unit(End - Begin, native, millisecond) + 50, %% Fail after 3 attempts. - RetryTimeout = 3 * max(MinDelay, get_retry_ms()), + RetryTimeout = ceil(3 * max(MinDelay, get_retry_ms())), OkOrFailed = case InitRes of {ok, _TnxId, _} when RequireNum =:= 1 -> @@ -334,9 +334,9 @@ log_and_alarm(false, Res, Meta) -> emqx_alarm:activate(cluster_rpc_apply_failed, NotOkMeta#{result => ?TO_BIN(Res)}). wait_for_all_nodes_commit(TnxId, Delay, Remain) -> - ok = timer:sleep(Delay), case lagging_node(TnxId) of [_ | _] when Remain > 0 -> + ok = timer:sleep(Delay), wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); [] -> ok; Nodes -> {error, Nodes} From 6da1196244b0db606296ec0a7cfdb088e288369d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Sep 2021 11:59:16 +0800 Subject: [PATCH 57/84] chore(gw-lwm2m): refine emqx_tlv_SUITE cases --- apps/emqx_gateway/test/emqx_tlv_SUITE.erl | 411 +++++++++++----------- 1 file changed, 205 insertions(+), 206 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index fd62ccf9d..a37eeb7d2 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -16,223 +16,222 @@ -module(emqx_tlv_SUITE). - -compile(export_all). - -compile(nowarn_export_all). +-compile(export_all). +-compile(nowarn_export_all). - -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). +-define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). - -include("src/lwm2m/include/emqx_lwm2m.hrl"). - -include_lib("lwm2m_coap/include/coap.hrl"). - -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. - all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09]. +case01(_Config) -> + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - init_per_suite(Config) -> - Config. +case02(_Config) -> + Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_multiple_resource => 16#06, value => [ + #{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>} + ]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - end_per_suite(Config) -> - Config. +case03(_Config) -> + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, + #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, + #{tlv_resource_with_value => 16#02, value => <<"345000123">>} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). +case03_0(_Config) -> + Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_multiple_resource => 16#02, value => [ + #{tlv_resource_instance => 16#7F, value => <<16#07>>}, + #{tlv_resource_instance => 16#0136, value => <<16#01>>} + ]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case01(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case04(_Config) -> + % 6.4.3.1 Single Object Instance Request Example + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, + #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, + #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, + #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, + #{tlv_multiple_resource => 16#06, value => [ + #{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>} + ]}, + #{tlv_multiple_resource => 16#07, value => [ + #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, + #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} + ]}, + #{tlv_multiple_resource => 16#08, value => [ + #{tlv_resource_instance => 16#00, value => <<16#7d>>}, + #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} + ]}, + #{tlv_resource_with_value => 16#09, value => <<16#64>>}, + #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, + #{tlv_multiple_resource => 16#0B, value => [ + #{tlv_resource_instance => 16#00, value => <<16#00>>} + ]}, + #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, + #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, + #{tlv_resource_with_value => 16#10, value => <<"U">>} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case02(_Config) -> - Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case05(_Config) -> + % 6.4.3.2 Multiple Object Instance Request Examples + % A) Request on Single-Instance Object + Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_object_instance => 16#00, value => [ + #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, + #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, + #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, + #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, + #{tlv_multiple_resource => 16#06, value => [ + #{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>} + ]}, + #{tlv_multiple_resource => 16#07, value => [ + #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, + #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} + ]}, + #{tlv_multiple_resource => 16#08, value => [ + #{tlv_resource_instance => 16#00, value => <<16#7d>>}, + #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} + ]}, + #{tlv_resource_with_value => 16#09, value => <<16#64>>}, + #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, + #{tlv_multiple_resource => 16#0B, value => [ + #{tlv_resource_instance => 16#00, value => <<16#00>>} + ]}, + #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, + #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, + #{tlv_resource_with_value => 16#10, value => <<"U">>} + ]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case03(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case06(_Config) -> + % 6.4.3.2 Multiple Object Instance Request Examples + % B) Request on Multiple-Instances Object having 2 instances + Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_object_instance => 16#00, value => [ + #{tlv_resource_with_value => 16#00, value => <<16#01>>}, + #{tlv_resource_with_value => 16#01, value => <<16#00>>}, + #{tlv_multiple_resource => 16#02, value => [ + #{tlv_resource_instance => 16#7F, value => <<16#07>>} + ]}, + #{tlv_resource_with_value => 16#03, value => <<16#7F>>} + ]}, + #{tlv_object_instance => 16#02, value => [ + #{tlv_resource_with_value => 16#00, value => <<16#03>>}, + #{tlv_resource_with_value => 16#01, value => <<16#00>>}, + #{tlv_multiple_resource => 16#02, value => [ + #{tlv_resource_instance => 16#7F, value => <<16#07>>}, + #{tlv_resource_instance => 16#0136, value => <<16#01>>} + ]}, + #{tlv_resource_with_value => 16#03, value => <<16#7F>>} + ]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case03_0(_Config) -> - Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case07(_Config) -> + % 6.4.3.2 Multiple Object Instance Request Examples + % C) Request on Multiple-Instances Object having 1 instance only + Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_object_instance => 16#00, value => [ + #{tlv_resource_with_value => 16#00, value => <<16#01>>}, + #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, + #{tlv_resource_with_value => 16#06, value => <<16#01>>}, + #{tlv_resource_with_value => 16#07, value => <<$U>>}]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case04(_Config) -> - % 6.4.3.1 Single Object Instance Request Example - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, - #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} - ]}, - #{tlv_resource_with_value => 16#09, value => <<16#64>>}, - #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, - #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, - #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, - #{tlv_resource_with_value => 16#10, value => <<"U">>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case08(_Config) -> + % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource + % Example 1) request to Object 65 Instance 0: Read /65/0 + Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_multiple_resource => 16#00, value => [ + #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, + #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>} + ]}, + #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, + #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). - case05(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % A) Request on Single-Instance Object - Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, - #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, - #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, - #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} - ]}, - #{tlv_resource_with_value => 16#09, value => <<16#64>>}, - #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, - #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, - #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, - #{tlv_resource_with_value => 16#10, value => <<"U">>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - - case06(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % B) Request on Multiple-Instances Object having 2 instances - Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]}, - #{tlv_object_instance => 16#02, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#03>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - - case07(_Config) -> - % 6.4.3.2 Multiple Object Instance Request Examples - % C) Request on Multiple-Instances Object having 1 instance only - Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, - #{tlv_resource_with_value => 16#06, value => <<16#01>>}, - #{tlv_resource_with_value => 16#07, value => <<$U>>}]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - - case08(_Config) -> - % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource - % Example 1) request to Object 65 Instance 0: Read /65/0 - Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#00, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, - #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>} - ]}, - #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, - #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). - - case09(_Config) -> - % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource - % Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances - Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>, - R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, - #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>} - ]}, - #{tlv_object_instance => 16#01, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, - #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} - ]} - ], - ?assertEqual(Exp, R), - EncodedBinary = emqx_lwm2m_tlv:encode(Exp), - ?assertEqual(EncodedBinary, Data). +case09(_Config) -> + % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource + % Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances + Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>, + R = emqx_lwm2m_tlv:parse(Data), + Exp = [ + #{tlv_object_instance => 16#00, value => [ + #{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, + #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, + #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>} + ]}, + #{tlv_object_instance => 16#01, value => [ + #{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, + #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, + #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} + ]} + ], + ?assertEqual(Exp, R), + EncodedBinary = emqx_lwm2m_tlv:encode(Exp), + ?assertEqual(EncodedBinary, Data). From cfabb8549e4943477ced2e62e9920d8f9664c2e9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Sep 2021 14:09:06 +0800 Subject: [PATCH 58/84] build(deps): upgrade emqx_http_lib to 0.4.1 --- apps/emqx/test/emqx_cm_locker_SUITE.erl | 8 ++++---- apps/emqx/test/emqx_shared_sub_SUITE.erl | 4 ++-- rebar.config | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/test/emqx_cm_locker_SUITE.erl b/apps/emqx/test/emqx_cm_locker_SUITE.erl index ec40a8985..90320a811 100644 --- a/apps/emqx/test/emqx_cm_locker_SUITE.erl +++ b/apps/emqx/test/emqx_cm_locker_SUITE.erl @@ -39,7 +39,7 @@ t_trans(_) -> ok = emqx_cm_locker:trans(<<"clientid">>, fun(_) -> ok end). t_lock_unlocak(_) -> - {true, _Nodes} = emqx_cm_locker:lock(<<"clientid">>), - {true, _Nodes} = emqx_cm_locker:lock(<<"clientid">>), - {true, _Nodes} = emqx_cm_locker:unlock(<<"clientid">>), - {true, _Nodes} = emqx_cm_locker:unlock(<<"clientid">>). + {true, _} = emqx_cm_locker:lock(<<"clientid">>), + {true, _} = emqx_cm_locker:lock(<<"clientid">>), + {true, _} = emqx_cm_locker:unlock(<<"clientid">>), + {true, _} = emqx_cm_locker:unlock(<<"clientid">>). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index e3caf44a7..17ed1026b 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -109,9 +109,9 @@ t_no_connection_nack(_) -> ExpProp = [{properties, #{'Session-Expiry-Interval' => timer:seconds(30)}}], {ok, SubConnPid1} = emqtt:start_link([{clientid, Subscriber1}] ++ ExpProp), - {ok, _Props} = emqtt:connect(SubConnPid1), + {ok, _Props1} = emqtt:connect(SubConnPid1), {ok, SubConnPid2} = emqtt:start_link([{clientid, Subscriber2}] ++ ExpProp), - {ok, _Props} = emqtt:connect(SubConnPid2), + {ok, _Props2} = emqtt:connect(SubConnPid2), emqtt:subscribe(SubConnPid1, ShareTopic, QoS), emqtt:subscribe(SubConnPid1, ShareTopic, QoS), diff --git a/rebar.config b/rebar.config index 1e31c68e4..310cd82ec 100644 --- a/rebar.config +++ b/rebar.config @@ -62,7 +62,7 @@ , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.1"}}} - , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} + , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} ]}. From d82830d8c7c1f26aee49e242604010c797874813 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Sep 2021 16:06:08 +0800 Subject: [PATCH 59/84] build(deps): upgrade er_coap_client to v1.0.4 --- rebar.config.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.erl b/rebar.config.erl index f818b4f84..4888fafe0 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -110,7 +110,7 @@ project_app_dirs() -> plugins(HasElixir) -> [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}} - , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.3"}}} + , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.4"}}} %% emqx main project does not require port-compiler %% pin at root level for deterministic , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} From 2fe96a3ad06b45b02409684b5eb65e539093bbb8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 17 Sep 2021 16:40:16 +0800 Subject: [PATCH 60/84] chore(gw-lwm2m): format emqx_lwm2m_tlv_SUITE --- apps/emqx_gateway/test/emqx_tlv_SUITE.erl | 236 ++++++++++++---------- 1 file changed, 134 insertions(+), 102 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl index a37eeb7d2..1191d748e 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_tlv_SUITE.erl @@ -25,7 +25,13 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -all() -> [case01, case02, case03, case03_0, case04, case05, case06, case07, case08, case09]. +%%-------------------------------------------------------------------- +%% Setup +%%-------------------------------------------------------------------- + +all() -> + [case01, case02, case03, case03_0, + case04, case05, case06, case07, case08, case09]. init_per_suite(Config) -> Config. @@ -33,9 +39,14 @@ init_per_suite(Config) -> end_per_suite(Config) -> Config. +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- case01(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, + 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, + 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65>>, R = emqx_lwm2m_tlv:parse(Data), Exp = [ #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>} @@ -47,24 +58,28 @@ case01(_Config) -> case02(_Config) -> Data = <<16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]} - ], + Exp = [ #{tlv_multiple_resource => 16#06, + value => [ #{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>}]} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). case03(_Config) -> - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33>>, + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, + 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, + 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, + 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, + 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, + 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, + 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, + 16#30, 16#31, 16#32, 16#33>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, + Exp = [ #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, #{tlv_resource_with_value => 16#02, value => <<"345000123">>} - ], + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). @@ -72,42 +87,46 @@ case03(_Config) -> case03_0(_Config) -> Data = <<16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]} - ], + Exp = [ #{tlv_multiple_resource => 16#02, + value => [ #{tlv_resource_instance => 16#7F, value => <<16#07>>}, + #{tlv_resource_instance => 16#0136, value => <<16#01>>} + ]} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). case04(_Config) -> % 6.4.3.1 Single Object Instance Request Example - Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, + Data = <<16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, + 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, + 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, + 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, + 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, + 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, + 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, + 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, + 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, + 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, + 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, + 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, + 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, + 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, + 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, + Exp = [ #{tlv_resource_with_value => 16#00, value => <<"Open Mobile Alliance">>}, #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} - ]}, + #{tlv_multiple_resource => 16#06, value => [#{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>}]}, + #{tlv_multiple_resource => 16#07, value => [#{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, + #{tlv_resource_instance => 16#01, value => <<16#1388:16>>}]}, + #{tlv_multiple_resource => 16#08, value => [#{tlv_resource_instance => 16#00, value => <<16#7d>>}, + #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}]}, #{tlv_resource_with_value => 16#09, value => <<16#64>>}, #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, + #{tlv_multiple_resource => 16#0B, value => [#{tlv_resource_instance => 16#00, value => <<16#00>>}]}, #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, #{tlv_resource_with_value => 16#10, value => <<"U">>} @@ -119,7 +138,22 @@ case04(_Config) -> case05(_Config) -> % 6.4.3.2 Multiple Object Instance Request Examples % A) Request on Single-Instance Object - Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, 16#30, 16#C1, 16#10, 16#55>>, + Data = <<16#08, 16#00, 16#79, 16#C8, 16#00, 16#14, 16#4F, 16#70, + 16#65, 16#6E, 16#20, 16#4D, 16#6F, 16#62, 16#69, 16#6C, + 16#65, 16#20, 16#41, 16#6C, 16#6C, 16#69, 16#61, 16#6E, + 16#63, 16#65, 16#C8, 16#01, 16#16, 16#4C, 16#69, 16#67, + 16#68, 16#74, 16#77, 16#65, 16#69, 16#67, 16#68, 16#74, + 16#20, 16#4D, 16#32, 16#4D, 16#20, 16#43, 16#6C, 16#69, + 16#65, 16#6E, 16#74, 16#C8, 16#02, 16#09, 16#33, 16#34, + 16#35, 16#30, 16#30, 16#30, 16#31, 16#32, 16#33, 16#C3, + 16#03, 16#31, 16#2E, 16#30, 16#86, 16#06, 16#41, 16#00, + 16#01, 16#41, 16#01, 16#05, 16#88, 16#07, 16#08, 16#42, + 16#00, 16#0E, 16#D8, 16#42, 16#01, 16#13, 16#88, 16#87, + 16#08, 16#41, 16#00, 16#7D, 16#42, 16#01, 16#03, 16#84, + 16#C1, 16#09, 16#64, 16#C1, 16#0A, 16#0F, 16#83, 16#0B, + 16#41, 16#00, 16#00, 16#C4, 16#0D, 16#51, 16#82, 16#42, + 16#8F, 16#C6, 16#0E, 16#2B, 16#30, 16#32, 16#3A, 16#30, + 16#30, 16#C1, 16#10, 16#55>>, R = emqx_lwm2m_tlv:parse(Data), Exp = [ #{tlv_object_instance => 16#00, value => [ @@ -127,23 +161,16 @@ case05(_Config) -> #{tlv_resource_with_value => 16#01, value => <<"Lightweight M2M Client">>}, #{tlv_resource_with_value => 16#02, value => <<"345000123">>}, #{tlv_resource_with_value => 16#03, value => <<"1.0">>}, - #{tlv_multiple_resource => 16#06, value => [ - #{tlv_resource_instance => 16#00, value => <<1>>}, - #{tlv_resource_instance => 16#01, value => <<5>>} - ]}, - #{tlv_multiple_resource => 16#07, value => [ - #{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, - #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} - ]}, - #{tlv_multiple_resource => 16#08, value => [ - #{tlv_resource_instance => 16#00, value => <<16#7d>>}, - #{tlv_resource_instance => 16#01, value => <<16#0384:16>>} + #{tlv_multiple_resource => 16#06, value => [#{tlv_resource_instance => 16#00, value => <<1>>}, + #{tlv_resource_instance => 16#01, value => <<5>>}]}, + #{tlv_multiple_resource => 16#07, value => [#{tlv_resource_instance => 16#00, value => <<16#0ED8:16>>}, + #{tlv_resource_instance => 16#01, value => <<16#1388:16>>} ]}, + #{tlv_multiple_resource => 16#08, value => [#{tlv_resource_instance => 16#00, value => <<16#7d>>}, + #{tlv_resource_instance => 16#01, value => <<16#0384:16>>}]}, #{tlv_resource_with_value => 16#09, value => <<16#64>>}, #{tlv_resource_with_value => 16#0A, value => <<16#0F>>}, - #{tlv_multiple_resource => 16#0B, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00>>} - ]}, + #{tlv_multiple_resource => 16#0B, value => [#{tlv_resource_instance => 16#00, value => <<16#00>>}]}, #{tlv_resource_with_value => 16#0D, value => <<16#5182428F:32>>}, #{tlv_resource_with_value => 16#0E, value => <<"+02:00">>}, #{tlv_resource_with_value => 16#10, value => <<"U">>} @@ -156,27 +183,25 @@ case05(_Config) -> case06(_Config) -> % 6.4.3.2 Multiple Object Instance Request Examples % B) Request on Multiple-Instances Object having 2 instances - Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, + Data = <<16#08, 16#00, 16#0E, 16#C1, 16#00, 16#01, 16#C1, 16#01, + 16#00, 16#83, 16#02, 16#41, 16#7F, 16#07, 16#C1, 16#03, + 16#7F, 16#08, 16#02, 16#12, 16#C1, 16#00, 16#03, 16#C1, + 16#01, 16#00, 16#87, 16#02, 16#41, 16#7F, 16#07, 16#61, + 16#01, 16#36, 16#01, 16#C1, 16#03, 16#7F>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]}, - #{tlv_object_instance => 16#02, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#03>>}, - #{tlv_resource_with_value => 16#01, value => <<16#00>>}, - #{tlv_multiple_resource => 16#02, value => [ - #{tlv_resource_instance => 16#7F, value => <<16#07>>}, - #{tlv_resource_instance => 16#0136, value => <<16#01>>} - ]}, - #{tlv_resource_with_value => 16#03, value => <<16#7F>>} - ]} - ], + Exp = [#{tlv_object_instance => 16#00, value => [#{tlv_resource_with_value => 16#00, value => <<16#01>>}, + #{tlv_resource_with_value => 16#01, value => <<16#00>>}, + #{tlv_multiple_resource => 16#02, + value => [#{tlv_resource_instance => 16#7F, value => <<16#07>>}]}, + #{tlv_resource_with_value => 16#03, value => <<16#7F>>} + ]}, + #{tlv_object_instance => 16#02, value => [#{tlv_resource_with_value => 16#00, value => <<16#03>>}, + #{tlv_resource_with_value => 16#01, value => <<16#00>>}, + #{tlv_multiple_resource => 16#02, + value => [#{tlv_resource_instance => 16#7F, value => <<16#07>>}, + #{tlv_resource_instance => 16#0136, value => <<16#01>>}]}, + #{tlv_resource_with_value => 16#03, value => <<16#7F>>}]} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). @@ -184,15 +209,16 @@ case06(_Config) -> case07(_Config) -> % 6.4.3.2 Multiple Object Instance Request Examples % C) Request on Multiple-Instances Object having 1 instance only - Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, 16#07, 16#55>>, + Data = <<16#08, 16#00, 16#0F, 16#C1, 16#00, 16#01, 16#C4, 16#01, + 16#00, 16#01, 16#51, 16#80, 16#C1, 16#06, 16#01, 16#C1, + 16#07, 16#55>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<16#01>>}, - #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, - #{tlv_resource_with_value => 16#06, value => <<16#01>>}, - #{tlv_resource_with_value => 16#07, value => <<$U>>}]} - ], + Exp = [#{tlv_object_instance => 16#00, + value => [#{tlv_resource_with_value => 16#00, value => <<16#01>>}, + #{tlv_resource_with_value => 16#01, value => <<86400:32>>}, + #{tlv_resource_with_value => 16#06, value => <<16#01>>}, + #{tlv_resource_with_value => 16#07, value => <<$U>>}]} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). @@ -200,16 +226,17 @@ case07(_Config) -> case08(_Config) -> % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource % Example 1) request to Object 65 Instance 0: Read /65/0 - Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, 16#02, 16#12, 16#34, 16#56, 16#78>>, + Data = <<16#88, 16#00, 16#0C, 16#44, 16#00, 16#00, 16#42, 16#00, + 16#00, 16#44, 16#01, 16#00, 16#42, 16#00, 16#01, 16#C8, + 16#01, 16#0D, 16#38, 16#36, 16#31, 16#33, 16#38, 16#30, + 16#30, 16#37, 16#35, 16#35, 16#35, 16#30, 16#30, 16#C4, + 16#02, 16#12, 16#34, 16#56, 16#78>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_multiple_resource => 16#00, value => [ - #{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, - #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>} - ]}, - #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, - #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} - ], + Exp = [#{tlv_multiple_resource => 16#00, value => [#{tlv_resource_instance => 16#00, value => <<16#00, 16#42, 16#00, 16#00>>}, + #{tlv_resource_instance => 16#01, value => <<16#00, 16#42, 16#00, 16#01>>}]}, + #{tlv_resource_with_value => 16#01, value => <<"8613800755500">>}, + #{tlv_resource_with_value => 16#02, value => <<16#12345678:32>>} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). @@ -217,21 +244,26 @@ case08(_Config) -> case09(_Config) -> % 6.4.3.3 Example of Request on an Object Instance containing an Object Link Resource % Example 2) request to Object 66: Read /66: TLV payload will contain 2 Object Instances - Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, 16#FF, 16#FF>>, + Data = <<16#08, 16#00, 16#26, 16#C8, 16#00, 16#0B, 16#6D, 16#79, + 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, 16#20, + 16#31, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, 16#65, + 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, 16#2E, + 16#32, 16#33, 16#34, 16#C4, 16#02, 16#00, 16#43, 16#00, + 16#00, 16#08, 16#01, 16#26, 16#C8, 16#00, 16#0B, 16#6D, + 16#79, 16#53, 16#65, 16#72, 16#76, 16#69, 16#63, 16#65, + 16#20, 16#32, 16#C8, 16#01, 16#0F, 16#49, 16#6E, 16#74, + 16#65, 16#72, 16#6E, 16#65, 16#74, 16#2E, 16#31, 16#35, + 16#2E, 16#32, 16#33, 16#35, 16#C4, 16#02, 16#FF, 16#FF, + 16#FF, 16#FF>>, R = emqx_lwm2m_tlv:parse(Data), - Exp = [ - #{tlv_object_instance => 16#00, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, - #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>} - ]}, - #{tlv_object_instance => 16#01, value => [ - #{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, - #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, - #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} - ]} - ], + Exp = [#{tlv_object_instance => 16#00, value => [#{tlv_resource_with_value => 16#00, value => <<"myService 1">>}, + #{tlv_resource_with_value => 16#01, value => <<"Internet.15.234">>}, + #{tlv_resource_with_value => 16#02, value => <<16#00, 16#43, 16#00, 16#00>>}]}, + #{tlv_object_instance => 16#01, value => [#{tlv_resource_with_value => 16#00, value => <<"myService 2">>}, + #{tlv_resource_with_value => 16#01, value => <<"Internet.15.235">>}, + #{tlv_resource_with_value => 16#02, value => <<16#FF, 16#FF, 16#FF, 16#FF>>} + ]} + ], ?assertEqual(Exp, R), EncodedBinary = emqx_lwm2m_tlv:encode(Exp), ?assertEqual(EncodedBinary, Data). - From 9c95557bfc11104447b560da974473d9bdeb813b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 17 Sep 2021 17:14:19 +0800 Subject: [PATCH 61/84] fix(emqx_types): type spec refs does not exist file. --- apps/emqx/test/emqx_request_handler.erl | 4 ++-- apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_request_handler.erl b/apps/emqx/test/emqx_request_handler.erl index ddd0a563f..62bcf4260 100644 --- a/apps/emqx/test/emqx_request_handler.erl +++ b/apps/emqx/test/emqx_request_handler.erl @@ -20,8 +20,8 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). --type qos() :: emqx_mqtt_types:qos_name() | emqx_mqtt_types:qos(). --type topic() :: emqx_topic:topic(). +-type qos() :: emqx_types:qos_name() | emqx_types:qos(). +-type topic() :: emqx_types:topic(). -type handler() :: fun((CorrData :: binary(), ReqPayload :: binary()) -> RspPayload :: binary()). -spec start_link(topic(), qos(), handler(), emqtt:options()) -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index bb505f98d..75bf88086 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -104,7 +104,7 @@ ]). -type id() :: atom() | string() | pid(). --type qos() :: emqx_mqtt_types:qos(). +-type qos() :: emqx_types:qos(). -type config() :: map(). -type batch() :: [emqx_connector_mqtt_msg:exp_msg()]. -type ack_ref() :: term(). diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 21c715745..f73858f32 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -265,7 +265,7 @@ payload(Path) -> %% @doc Check if a topic_filter contains a specific topic %% TopicFilters = [{<<"t/a">>, #{qos => 0}]. --spec(contains_topic(emqx_mqtt_types:topic_filters(), emqx_types:topic()) +-spec(contains_topic(emqx_types:topic_filters(), emqx_types:topic()) -> true | false). contains_topic(TopicFilters, Topic) -> case find_topic_filter(Topic, TopicFilters, fun eq/2) of @@ -278,7 +278,7 @@ contains_topic(TopicFilters, Topic, QoS) -> _ -> false end. --spec(contains_topic_match(emqx_mqtt_types:topic_filters(), emqx_types:topic()) +-spec(contains_topic_match(emqx_types:topic_filters(), emqx_types:topic()) -> true | false). contains_topic_match(TopicFilters, Topic) -> case find_topic_filter(Topic, TopicFilters, fun emqx_topic:match/2) of From 6edc9f4221c1941c28038ccd60f8e97f72c57bb1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 17 Sep 2021 17:15:32 +0800 Subject: [PATCH 62/84] fix(emqx_types): spec refs `emqx_types.erl` instead directly. --- apps/emqx/src/emqx.erl | 16 ++++++------ apps/emqx/src/emqx_authz_cache.erl | 4 +-- apps/emqx/src/emqx_broker.erl | 22 ++++++++-------- apps/emqx/src/emqx_broker_helper.erl | 6 ++--- apps/emqx/src/emqx_message.erl | 10 +++---- apps/emqx/src/emqx_mqtt_caps.erl | 2 +- apps/emqx/src/emqx_mqueue.erl | 2 +- apps/emqx/src/emqx_router.erl | 26 +++++++++---------- apps/emqx/src/emqx_shared_sub.erl | 6 ++--- apps/emqx/src/emqx_trie.erl | 6 ++--- apps/emqx/src/emqx_types.erl | 7 +++-- apps/emqx_authz/include/emqx_authz.hrl | 2 +- apps/emqx_authz/src/emqx_authz.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 6 ++--- 14 files changed, 58 insertions(+), 59 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 1d4686561..e64899b5b 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -119,17 +119,17 @@ is_running(Node) -> %% PubSub API %%-------------------------------------------------------------------- --spec(subscribe(emqx_topic:topic() | string()) -> ok). +-spec(subscribe(emqx_types:topic() | string()) -> ok). subscribe(Topic) -> emqx_broker:subscribe(iolist_to_binary(Topic)). --spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok). +-spec(subscribe(emqx_types:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId)-> emqx_broker:subscribe(iolist_to_binary(Topic), SubId); subscribe(Topic, SubOpts) when is_map(SubOpts) -> emqx_broker:subscribe(iolist_to_binary(Topic), SubOpts). --spec(subscribe(emqx_topic:topic() | string(), +-spec(subscribe(emqx_types:topic() | string(), emqx_types:subid() | pid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), is_map(SubOpts) -> emqx_broker:subscribe(iolist_to_binary(Topic), SubId, SubOpts). @@ -138,7 +138,7 @@ subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), publish(Msg) -> emqx_broker:publish(Msg). --spec(unsubscribe(emqx_topic:topic() | string()) -> ok). +-spec(unsubscribe(emqx_types:topic() | string()) -> ok). unsubscribe(Topic) -> emqx_broker:unsubscribe(iolist_to_binary(Topic)). @@ -146,18 +146,18 @@ unsubscribe(Topic) -> %% PubSub management API %%-------------------------------------------------------------------- --spec(topics() -> list(emqx_topic:topic())). +-spec(topics() -> list(emqx_types:topic())). topics() -> emqx_router:topics(). --spec(subscribers(emqx_topic:topic() | string()) -> [pid()]). +-spec(subscribers(emqx_types:topic() | string()) -> [pid()]). subscribers(Topic) -> emqx_broker:subscribers(iolist_to_binary(Topic)). --spec(subscriptions(pid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). +-spec(subscriptions(pid()) -> [{emqx_types:topic(), emqx_types:subopts()}]). subscriptions(SubPid) when is_pid(SubPid) -> emqx_broker:subscriptions(SubPid). --spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic() | string()) -> boolean()). +-spec(subscribed(pid() | emqx_types:subid(), emqx_types:topic() | string()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> emqx_broker:subscribed(SubPid, iolist_to_binary(Topic)); subscribed(SubId, Topic) when is_atom(SubId); is_binary(SubId) -> diff --git a/apps/emqx/src/emqx_authz_cache.erl b/apps/emqx/src/emqx_authz_cache.erl index 10ddbd21c..43807a922 100644 --- a/apps/emqx/src/emqx_authz_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -68,7 +68,7 @@ list_authz_cache() -> map_authz_cache(fun(Cache) -> Cache end). %% We'll cleanup the cache before replacing an expired authz. --spec get_authz_cache(emqx_types:pubsub(), emqx_topic:topic()) -> +-spec get_authz_cache(emqx_types:pubsub(), emqx_types:topic()) -> authz_result() | not_found. get_authz_cache(PubSub, Topic) -> case erlang:get(cache_k(PubSub, Topic)) of @@ -85,7 +85,7 @@ get_authz_cache(PubSub, Topic) -> %% If the cache get full, and also the latest one %% is expired, then delete all the cache entries --spec put_authz_cache(emqx_types:pubsub(), emqx_topic:topic(), authz_result()) +-spec put_authz_cache(emqx_types:pubsub(), emqx_types:topic(), authz_result()) -> ok. put_authz_cache(PubSub, Topic, AuthzResult) -> MaxSize = get_cache_max_size(), true = (MaxSize =/= 0), diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 46accb9fe..56ac348da 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -112,17 +112,17 @@ create_tabs() -> %% Subscribe API %%------------------------------------------------------------------------------ --spec(subscribe(emqx_topic:topic()) -> ok). +-spec(subscribe(emqx_types:topic()) -> ok). subscribe(Topic) when is_binary(Topic) -> subscribe(Topic, undefined). --spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). +-spec(subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). --spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). +-spec(subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), case ets:member(?SUBOPTION, {SubPid = self(), Topic}) of @@ -165,7 +165,7 @@ do_subscribe(Group, Topic, SubPid, SubOpts) -> %% Unsubscribe API %%-------------------------------------------------------------------- --spec(unsubscribe(emqx_topic:topic()) -> ok). +-spec(unsubscribe(emqx_types:topic()) -> ok). unsubscribe(Topic) when is_binary(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {SubPid, Topic}) of @@ -279,7 +279,7 @@ forward(Node, To, Delivery, sync) -> emqx_metrics:inc('messages.forward'), Result end. --spec(dispatch(emqx_topic:topic(), emqx_types:delivery()) -> emqx_types:deliver_result()). +-spec(dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result()). dispatch(Topic, #delivery{message = Msg}) -> DispN = lists:foldl( fun(Sub, N) -> @@ -316,7 +316,7 @@ inc_dropped_cnt(Msg) -> end. -compile({inline, [subscribers/1]}). --spec(subscribers(emqx_topic:topic() | {shard, emqx_topic:topic(), non_neg_integer()}) +-spec(subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) -> [pid()]). subscribers(Topic) when is_binary(Topic) -> lookup_value(?SUBSCRIBER, Topic, []); @@ -351,7 +351,7 @@ subscriber_down(SubPid) -> %%-------------------------------------------------------------------- -spec(subscriptions(pid() | emqx_types:subid()) - -> [{emqx_topic:topic(), emqx_types:subopts()}]). + -> [{emqx_types:topic(), emqx_types:subopts()}]). subscriptions(SubPid) when is_pid(SubPid) -> [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})} || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])]; @@ -362,14 +362,14 @@ subscriptions(SubId) -> undefined -> [] end. --spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic()) -> boolean()). +-spec(subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean()). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {SubPid, Topic}); subscribed(SubId, Topic) when ?is_subid(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {SubPid, Topic}). --spec(get_subopts(pid(), emqx_topic:topic()) -> maybe(emqx_types:subopts())). +-spec(get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts())). get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> lookup_value(?SUBOPTION, {SubPid, Topic}); get_subopts(SubId, Topic) when ?is_subid(SubId) -> @@ -379,7 +379,7 @@ get_subopts(SubId, Topic) when ?is_subid(SubId) -> undefined -> undefined end. --spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). +-spec(set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean()). set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> set_subopts(self(), Topic, NewOpts). @@ -392,7 +392,7 @@ set_subopts(SubPid, Topic, NewOpts) -> [] -> false end. --spec(topics() -> [emqx_topic:topic()]). +-spec(topics() -> [emqx_types:topic()]). topics() -> emqx_router:topics(). diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index fad4e8713..fdd1a55d9 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -78,7 +78,7 @@ lookup_subid(SubPid) when is_pid(SubPid) -> lookup_subpid(SubId) -> emqx_tables:lookup_value(?SUBID, SubId). --spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +-spec(get_sub_shard(pid(), emqx_types:topic()) -> non_neg_integer()). get_sub_shard(SubPid, Topic) -> case create_seq(Topic) of Seq when Seq =< ?SHARD -> 0; @@ -90,11 +90,11 @@ shards_num() -> %% Dynamic sharding later... ets:lookup_element(?HELPER, shards, 2). --spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +-spec(create_seq(emqx_types:topic()) -> emqx_sequence:seqid()). create_seq(Topic) -> emqx_sequence:nextval(?SUBSEQ, Topic). --spec(reclaim_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +-spec(reclaim_seq(emqx_types:topic()) -> emqx_sequence:seqid()). reclaim_seq(Topic) -> emqx_sequence:reclaim(?SUBSEQ, Topic). diff --git a/apps/emqx/src/emqx_message.erl b/apps/emqx/src/emqx_message.erl index faae621d8..b70655fc5 100644 --- a/apps/emqx/src/emqx_message.erl +++ b/apps/emqx/src/emqx_message.erl @@ -86,19 +86,19 @@ -elvis([{elvis_style, god_modules, disable}]). --spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). +-spec(make(emqx_types:topic(), emqx_types:payload()) -> emqx_types:message()). make(Topic, Payload) -> make(undefined, Topic, Payload). -spec(make(emqx_types:clientid(), - emqx_topic:topic(), + emqx_types:topic(), emqx_types:payload()) -> emqx_types:message()). make(From, Topic, Payload) -> make(From, ?QOS_0, Topic, Payload). -spec(make(emqx_types:clientid(), emqx_types:qos(), - emqx_topic:topic(), + emqx_types:topic(), emqx_types:payload()) -> emqx_types:message()). make(From, QoS, Topic, Payload) when ?QOS_0 =< QoS, QoS =< ?QOS_2 -> Now = erlang:system_time(millisecond), @@ -112,7 +112,7 @@ make(From, QoS, Topic, Payload) when ?QOS_0 =< QoS, QoS =< ?QOS_2 -> -spec(make(emqx_types:clientid(), emqx_types:qos(), - emqx_topic:topic(), + emqx_types:topic(), emqx_types:payload(), emqx_types:flags(), emqx_types:headers()) -> emqx_types:message()). @@ -133,7 +133,7 @@ make(From, QoS, Topic, Payload, Flags, Headers) -spec(make(MsgId :: binary(), emqx_types:clientid(), emqx_types:qos(), - emqx_topic:topic(), + emqx_types:topic(), emqx_types:payload(), emqx_types:flags(), emqx_types:headers()) -> emqx_types:message()). diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index add86ef99..a1da0c98e 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -67,7 +67,7 @@ -spec(check_pub(emqx_types:zone(), #{qos := emqx_types:qos(), retain := boolean(), - topic := emqx_topic:topic()}) + topic := emqx_types:topic()}) -> ok_or_error(emqx_types:reason_code())). check_pub(Zone, Flags) when is_map(Flags) -> do_check_pub(case maps:take(topic, Flags) of diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl index 259c5c428..8ccbc8d56 100644 --- a/apps/emqx/src/emqx_mqueue.erl +++ b/apps/emqx/src/emqx_mqueue.erl @@ -71,7 +71,7 @@ -export_type([mqueue/0, options/0]). --type(topic() :: emqx_topic:topic()). +-type(topic() :: emqx_types:topic()). -type(priority() :: infinity | integer()). -type(pq() :: emqx_pqueue:q()). -type(count() :: non_neg_integer()). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index c39571d9f..d25a8bec6 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -98,19 +98,19 @@ start_link(Pool, Id) -> %% Route APIs %%-------------------------------------------------------------------- --spec(add_route(emqx_topic:topic()) -> ok | {error, term()}). +-spec(add_route(emqx_types:topic()) -> ok | {error, term()}). add_route(Topic) when is_binary(Topic) -> add_route(Topic, node()). --spec(add_route(emqx_topic:topic(), dest()) -> ok | {error, term()}). +-spec(add_route(emqx_types:topic(), dest()) -> ok | {error, term()}). add_route(Topic, Dest) when is_binary(Topic) -> call(pick(Topic), {add_route, Topic, Dest}). --spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}). +-spec(do_add_route(emqx_types:topic()) -> ok | {error, term()}). do_add_route(Topic) when is_binary(Topic) -> do_add_route(Topic, node()). --spec(do_add_route(emqx_topic:topic(), dest()) -> ok | {error, term()}). +-spec(do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}). do_add_route(Topic, Dest) when is_binary(Topic) -> Route = #route{topic = Topic, dest = Dest}, case lists:member(Route, lookup_routes(Topic)) of @@ -125,7 +125,7 @@ do_add_route(Topic, Dest) when is_binary(Topic) -> end. %% @doc Match routes --spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). +-spec(match_routes(emqx_types:topic()) -> [emqx_types:route()]). match_routes(Topic) when is_binary(Topic) -> case match_trie(Topic) of [] -> lookup_routes(Topic); @@ -140,27 +140,27 @@ match_trie(Topic) -> false -> emqx_trie:match(Topic) end. --spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). +-spec(lookup_routes(emqx_types:topic()) -> [emqx_types:route()]). lookup_routes(Topic) -> ets:lookup(?ROUTE_TAB, Topic). --spec(has_routes(emqx_topic:topic()) -> boolean()). +-spec(has_routes(emqx_types:topic()) -> boolean()). has_routes(Topic) when is_binary(Topic) -> ets:member(?ROUTE_TAB, Topic). --spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}). +-spec(delete_route(emqx_types:topic()) -> ok | {error, term()}). delete_route(Topic) when is_binary(Topic) -> delete_route(Topic, node()). --spec(delete_route(emqx_topic:topic(), dest()) -> ok | {error, term()}). +-spec(delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}). delete_route(Topic, Dest) when is_binary(Topic) -> call(pick(Topic), {delete_route, Topic, Dest}). --spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}). +-spec(do_delete_route(emqx_types:topic()) -> ok | {error, term()}). do_delete_route(Topic) when is_binary(Topic) -> do_delete_route(Topic, node()). --spec(do_delete_route(emqx_topic:topic(), dest()) -> ok | {error, term()}). +-spec(do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}). do_delete_route(Topic, Dest) -> Route = #route{topic = Topic, dest = Dest}, case emqx_topic:wildcard(Topic) of @@ -169,12 +169,12 @@ do_delete_route(Topic, Dest) -> false -> delete_direct_route(Route) end. --spec(topics() -> list(emqx_topic:topic())). +-spec(topics() -> list(emqx_types:topic())). topics() -> mnesia:dirty_all_keys(?ROUTE_TAB). %% @doc Print routes to a topic --spec(print_routes(emqx_topic:topic()) -> ok). +-spec(print_routes(emqx_types:topic()) -> ok). print_routes(Topic) -> lists:foreach(fun(#route{topic = To, dest = Dest}) -> io:format("~s -> ~s~n", [To, Dest]) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 9e5dd726f..59e364f58 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -103,18 +103,18 @@ mnesia(copy) -> start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). --spec(subscribe(emqx_topic:group(), emqx_topic:topic(), pid()) -> ok). +-spec(subscribe(emqx_types:group(), emqx_types:topic(), pid()) -> ok). subscribe(Group, Topic, SubPid) when is_pid(SubPid) -> gen_server:call(?SERVER, {subscribe, Group, Topic, SubPid}). --spec(unsubscribe(emqx_topic:group(), emqx_topic:topic(), pid()) -> ok). +-spec(unsubscribe(emqx_types:group(), emqx_types:topic(), pid()) -> ok). unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) -> gen_server:call(?SERVER, {unsubscribe, Group, Topic, SubPid}). record(Group, Topic, SubPid) -> #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}. --spec(dispatch(emqx_topic:group(), emqx_topic:topic(), emqx_types:delivery()) +-spec(dispatch(emqx_types:group(), emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result()). dispatch(Group, Topic, Delivery) -> dispatch(Group, Topic, Delivery, _FailedSubs = []). diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index ea70ff7f3..1e3a0e5a5 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -77,7 +77,7 @@ mnesia(copy) -> %%-------------------------------------------------------------------- %% @doc Insert a topic filter into the trie. --spec(insert(emqx_topic:topic()) -> ok). +-spec(insert(emqx_types:topic()) -> ok). insert(Topic) when is_binary(Topic) -> {TopicKey, PrefixKeys} = make_keys(Topic), case mnesia:wread({?TRIE, TopicKey}) of @@ -86,7 +86,7 @@ insert(Topic) when is_binary(Topic) -> end. %% @doc Delete a topic filter from the trie. --spec(delete(emqx_topic:topic()) -> ok). +-spec(delete(emqx_types:topic()) -> ok). delete(Topic) when is_binary(Topic) -> {TopicKey, PrefixKeys} = make_keys(Topic), case [] =/= mnesia:wread({?TRIE, TopicKey}) of @@ -95,7 +95,7 @@ delete(Topic) when is_binary(Topic) -> end. %% @doc Find trie nodes that matchs the topic name. --spec(match(emqx_topic:topic()) -> list(emqx_topic:topic())). +-spec(match(emqx_types:topic()) -> list(emqx_types:topic())). match(Topic) when is_binary(Topic) -> Words = emqx_topic:words(Topic), case emqx_topic:wildcard(Words) of diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 940c9b630..ed17a59e4 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -146,7 +146,7 @@ dn => binary(), atom() => term() }). --type(clientid() :: binary()|atom()). +-type(clientid() :: binary() | atom()). -type(username() :: maybe(binary())). -type(password() :: maybe(binary())). -type(peerhost() :: inet:ip_address()). @@ -201,8 +201,8 @@ -type(publish_result() :: [{node(), topic(), deliver_result()} | {share, topic(), deliver_result()}]). -type(route() :: #route{}). --type(sub_group() :: tuple() | binary()). --type(route_entry() :: {topic(), node()} | {topic, sub_group()}). +-type(group() :: emqx_topic:group()). +-type(route_entry() :: {topic(), node()} | {topic, group()}). -type(plugin() :: #plugin{}). -type(command() :: #command{}). @@ -215,4 +215,3 @@ max_heap_size => non_neg_integer(), enable => boolean() }). - diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 83d7601c6..c83dfde0d 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -14,7 +14,7 @@ -type(permission() :: allow | deny). --type(rule() :: {permission(), who(), action(), list(emqx_topic:topic())}). +-type(rule() :: {permission(), who(), action(), list(emqx_types:topic())}). -type(rules() :: [rule()]). -type(sources() :: [map()]). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index b4debf3fb..cf7447f6a 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -326,7 +326,7 @@ init_source(#{enable := false} = Source) ->Source. %%-------------------------------------------------------------------- %% @doc Check AuthZ --spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_topic:topic(), allow | deny, sources()) +-spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_types:topic(), allow | deny, sources()) -> {stop, allow} | {ok, deny}). authorize(#{username := Username, peerhost := IpAddress diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 75bf88086..e5a1a807f 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -108,7 +108,7 @@ -type config() :: map(). -type batch() :: [emqx_connector_mqtt_msg:exp_msg()]. -type ack_ref() :: term(). --type topic() :: emqx_topic:topic(). +-type topic() :: emqx_types:topic(). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -176,7 +176,7 @@ ping(Name) -> get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(1000)). %% @doc Return all subscriptions (subscription over mqtt connection to remote broker). --spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. +-spec get_subscriptions(id()) -> [{emqx_types:topic(), qos()}]. get_subscriptions(Name) -> gen_statem:call(name(Name), get_subscriptions). callback_mode() -> [state_functions]. @@ -532,4 +532,4 @@ str(A) when is_atom(A) -> str(B) when is_binary(B) -> binary_to_list(B); str(S) when is_list(S) -> - S. \ No newline at end of file + S. From 35a4a05f0380de8a473c29a0fc9c234dbe69e404 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 16 Sep 2021 15:13:38 +0800 Subject: [PATCH 63/84] feat(clear certs): clear certs when deleting instance --- apps/emqx/src/emqx_authentication.erl | 80 ++++++++++++++------ apps/emqx/test/emqx_authentication_SUITE.erl | 19 +++-- apps/emqx_authn/src/emqx_authn_api.erl | 5 ++ 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index f047a1e41..8202ca2b4 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -199,12 +199,15 @@ pre_config_update(UpdateReq, OldConfig) -> {ok, NewConfig} -> {ok, may_to_map(NewConfig)} end. -do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> - try convert_certs(Config) of - NConfig -> - {ok, OldConfig ++ [NConfig]} +do_pre_config_update({create_authenticator, ChainName, Config}, OldConfig) -> + try + CertsDir = certs_dir([to_bin(ChainName), generate_id(Config)]), + NConfig = convert_certs(CertsDir, Config), + {ok, OldConfig ++ [NConfig]} catch error:{save_cert_to_file, _} = Reason -> + {error, Reason}; + error:{missing_parameter, _} = Reason -> {error, Reason} end; do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> @@ -212,17 +215,21 @@ do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldCon AuthenticatorID =/= generate_id(OldConfig0) end, OldConfig), {ok, NewConfig}; -do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> - try lists:map(fun(OldConfig0) -> - case AuthenticatorID =:= generate_id(OldConfig0) of - true -> convert_certs(Config, OldConfig0); - false -> OldConfig0 - end - end, OldConfig) of - NewConfig -> - {ok, NewConfig} +do_pre_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, OldConfig) -> + try + CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), + NewConfig = lists:map( + fun(OldConfig0) -> + case AuthenticatorID =:= generate_id(OldConfig0) of + true -> convert_certs(CertsDir, Config, OldConfig0); + false -> OldConfig0 + end + end, OldConfig), + {ok, NewConfig} catch error:{save_cert_to_file, _} = Reason -> + {error, Reason}; + error:{missing_parameter, _} = Reason -> {error, Reason} end; do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> @@ -254,8 +261,16 @@ do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _Ol _ = create_chain(ChainName), create_authenticator(ChainName, NConfig); -do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, _OldConfig, _AppEnvs) -> - delete_authenticator(ChainName, AuthenticatorID); +do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, OldConfig, _AppEnvs) -> + case delete_authenticator(ChainName, AuthenticatorID) of + ok -> + [Config] = [Config0 || Config0 <- OldConfig, AuthenticatorID == generate_id(Config0)], + CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), + clear_certs(CertsDir, Config), + ok; + {error, Reason} -> + {error, Reason} + end; do_post_config_update({update_authenticator, ChainName, AuthenticatorID, _Config}, NewConfig, _OldConfig, _AppEnvs) -> [Config] = lists:filter(fun(NewConfig0) -> @@ -449,7 +464,9 @@ generate_id(#{mechanism := Mechanism}) -> generate_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> <>; generate_id(#{<<"mechanism">> := Mechanism}) -> - Mechanism. + Mechanism; +generate_id(_) -> + error({missing_parameter, mechanism}). %%-------------------------------------------------------------------- %% gen_server callbacks @@ -642,34 +659,46 @@ reply(Reply, State) -> %% Internal functions %%------------------------------------------------------------------------------ -convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> +certs_dir(Dirs) when is_list(Dirs) -> + to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn"] ++ Dirs)). + +convert_certs(CertsDir, #{<<"ssl">> := SSLOpts} = Config) -> NSSLOPts = lists:foldl(fun(K, Acc) -> case maps:get(K, Acc, undefined) of undefined -> Acc; PemBin -> - CertFile = generate_filename(K), + CertFile = generate_filename(CertsDir, K), ok = save_cert_to_file(CertFile, PemBin), Acc#{K => CertFile} end end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), Config#{<<"ssl">> => NSSLOPts}; -convert_certs(Config) -> +convert_certs(_CertsDir, Config) -> Config. -convert_certs(#{<<"ssl">> := NewSSLOpts} = NewConfig, OldConfig) -> +convert_certs(CertsDir, #{<<"ssl">> := NewSSLOpts} = NewConfig, OldConfig) -> OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), Diff = diff_certs(NewSSLOpts, OldSSLOpts), NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> Acc#{K => maps:get(K, OldSSLOpts)}; ({_, K}, Acc) -> - CertFile = generate_filename(K), + CertFile = generate_filename(CertsDir, K), ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), Acc#{K => CertFile} end, NewSSLOpts, Diff), NewConfig#{<<"ssl">> => NSSLOpts}; -convert_certs(NewConfig, _OldConfig) -> +convert_certs(_CertsDir, NewConfig, _OldConfig) -> NewConfig. +clear_certs(CertsDir, #{<<"ssl">> := SSLOpts}) -> + lists:foreach( + fun({_, Filename}) -> + _ = file:delete(filename:join([CertsDir, Filename])) + end, + maps:to_list(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], SSLOpts))); +clear_certs(_CertsDir, _Config) -> + ok. + save_cert_to_file(Filename, PemBin) -> case public_key:pem_decode(PemBin) =/= [] of true -> @@ -686,13 +715,13 @@ save_cert_to_file(Filename, PemBin) -> error({save_cert_to_file, invalid_certificate}) end. -generate_filename(Key) -> +generate_filename(CertsDir, Key) -> Prefix = case Key of <<"keyfile">> -> "key-"; <<"certfile">> -> "cert-"; <<"cacertfile">> -> "cacert-" end, - to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn", Prefix ++ emqx_misc:gen_id() ++ ".pem"])). + to_bin(filename:join([CertsDir, Prefix ++ emqx_misc:gen_id() ++ ".pem"])). diff_certs(NewSSLOpts, OldSSLOpts) -> Keys = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>], @@ -900,6 +929,7 @@ to_list(L) when is_list(L) -> L. to_bin(B) when is_binary(B) -> B; -to_bin(L) when is_list(L) -> list_to_binary(L). +to_bin(L) when is_list(L) -> list_to_binary(L); +to_bin(A) when is_atom(A) -> atom_to_binary(A). call(Call) -> gen_server:call(?MODULE, Call, infinity). diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 15aecc269..2184cbba8 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -257,19 +257,22 @@ t_update_config({'end', Config}) -> ?AUTHN:deregister_providers([?config("auth1"), ?config("auth2")]), ok. -t_convert_cert_options({_, Config}) -> Config; -t_convert_cert_options(Config) when is_list(Config) -> +t_convert_certs({_, Config}) -> Config; +t_convert_certs(Config) when is_list(Config) -> + Global = <<"mqtt:global">>, Certs = certs([ {<<"keyfile">>, "key.pem"} , {<<"certfile">>, "cert.pem"} , {<<"cacertfile">>, "cacert.pem"} ]), - #{<<"ssl">> := NCerts} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs}), + + CertsDir = ?AUTHN:certs_dir([Global, <<"password-based:built-in-database">>]), + #{<<"ssl">> := NCerts} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, Certs))), Certs2 = certs([ {<<"keyfile">>, "key.pem"} , {<<"certfile">>, "cert.pem"} ]), - #{<<"ssl">> := NCerts2} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs2}, #{<<"ssl">> => NCerts}), + #{<<"ssl">> := NCerts2} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs2}, #{<<"ssl">> => NCerts}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, Certs2))), ?assertEqual(maps:get(<<"keyfile">>, NCerts), maps:get(<<"keyfile">>, NCerts2)), ?assertEqual(maps:get(<<"certfile">>, NCerts), maps:get(<<"certfile">>, NCerts2)), @@ -278,10 +281,14 @@ t_convert_cert_options(Config) when is_list(Config) -> , {<<"certfile">>, "client-cert.pem"} , {<<"cacertfile">>, "cacert.pem"} ]), - #{<<"ssl">> := NCerts3} = ?AUTHN:convert_certs(#{<<"ssl">> => Certs3}, #{<<"ssl">> => NCerts2}), + #{<<"ssl">> := NCerts3} = ?AUTHN:convert_certs(CertsDir, #{<<"ssl">> => Certs3}, #{<<"ssl">> => NCerts2}), ?assertEqual(false, diff_cert(maps:get(<<"keyfile">>, NCerts3), maps:get(<<"keyfile">>, Certs3))), ?assertNotEqual(maps:get(<<"keyfile">>, NCerts2), maps:get(<<"keyfile">>, NCerts3)), - ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)). + ?assertNotEqual(maps:get(<<"certfile">>, NCerts2), maps:get(<<"certfile">>, NCerts3)), + + ?assertEqual(true, filelib:is_regular(maps:get(<<"keyfile">>, NCerts3))), + ?AUTHN:clear_certs(CertsDir, #{<<"ssl">> => NCerts3}), + ?assertEqual(false, filelib:is_regular(maps:get(<<"keyfile">>, NCerts3))). update_config(Path, ConfigRequest) -> emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 93e8c4746..0eb2f5cfa 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1589,6 +1589,11 @@ definitions() -> type => string, example => <<"http://localhost:80">> }, + refresh_interval => #{ + type => integer, + default => 300, + example => 300 + }, verify_claims => #{ type => object, additionalProperties => #{ From b89973ce7cc98e7cce56deccf035109bbc0f1fff Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 16 Sep 2021 16:00:51 +0800 Subject: [PATCH 64/84] fix(authn): fix type error --- apps/emqx/src/emqx_authentication.erl | 8 ++++---- apps/emqx/test/emqx_authentication_SUITE.erl | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 8202ca2b4..ecef32475 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -264,7 +264,7 @@ do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _Ol do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, OldConfig, _AppEnvs) -> case delete_authenticator(ChainName, AuthenticatorID) of ok -> - [Config] = [Config0 || Config0 <- OldConfig, AuthenticatorID == generate_id(Config0)], + [Config] = [Config0 || Config0 <- to_list(OldConfig), AuthenticatorID == generate_id(Config0)], CertsDir = certs_dir([to_bin(ChainName), AuthenticatorID]), clear_certs(CertsDir, Config), ok; @@ -456,11 +456,11 @@ list_users(ChainName, AuthenticatorID) -> -spec generate_id(config()) -> authenticator_id(). generate_id(#{mechanism := Mechanism0, backend := Backend0}) -> - Mechanism = atom_to_binary(Mechanism0), - Backend = atom_to_binary(Backend0), + Mechanism = to_bin(Mechanism0), + Backend = to_bin(Backend0), <>; generate_id(#{mechanism := Mechanism}) -> - atom_to_binary(Mechanism); + to_bin(Mechanism); generate_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> <>; generate_id(#{<<"mechanism">> := Mechanism}) -> diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 2184cbba8..5fd2e47af 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -211,12 +211,12 @@ t_update_config(Config) when is_list(Config) -> ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), - AuthenticatorConfig1 = #{mechanism => 'password-based', - backend => 'built-in-database', - enable => true}, - AuthenticatorConfig2 = #{mechanism => 'password-based', - backend => mysql, - enable => true}, + AuthenticatorConfig1 = #{<<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"built-in-database">>, + <<"enable">> => true}, + AuthenticatorConfig2 = #{<<"mechanism">> => <<"password-based">>, + <<"backend">> => <<"mysql">>, + <<"enable">> => true}, ID1 = <<"password-based:built-in-database">>, ID2 = <<"password-based:mysql">>, @@ -227,7 +227,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID2)), - ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, AuthenticatorConfig1#{enable => false}})), + ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, AuthenticatorConfig1#{<<"enable">> => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch({ok, _}, update_config([authentication], {move_authenticator, Global, ID2, top})), @@ -244,7 +244,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig2})), ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID2)), - ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, AuthenticatorConfig1#{enable => false}})), + ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, AuthenticatorConfig1#{<<"enable">> => false}})), ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), ?assertMatch({ok, _}, update_config(ConfKeyPath, {move_authenticator, ListenerID, ID2, top})), From 69755ad3fb2d2d52359e3a01ff6792652dd77779 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 16 Sep 2021 16:45:56 +0800 Subject: [PATCH 65/84] fix(authn): fix bug in authn --- apps/emqx/src/emqx_authentication.erl | 5 +---- apps/emqx_authn/src/simple_authn/emqx_authn_http.erl | 4 ++-- apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl | 8 ++------ apps/emqx_connector/src/emqx_connector_http.erl | 6 +++--- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index ecef32475..8c682d1fc 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -272,10 +272,7 @@ do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewCo {error, Reason} end; -do_post_config_update({update_authenticator, ChainName, AuthenticatorID, _Config}, NewConfig, _OldConfig, _AppEnvs) -> - [Config] = lists:filter(fun(NewConfig0) -> - AuthenticatorID =:= generate_id(NewConfig0) - end, NewConfig), +do_post_config_update({update_authenticator, ChainName, AuthenticatorID, Config}, _NewConfig, _OldConfig, _AppEnvs) -> NConfig = check_config(Config), update_authenticator(ChainName, AuthenticatorID, NConfig); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index daa7f8073..9814d3e58 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -78,7 +78,7 @@ validations() -> url(type) -> binary(); url(nullable) -> false; -url(validate) -> [fun check_url/1]; +url(validator) -> [fun check_url/1]; url(_) -> undefined. headers(type) -> map(); @@ -99,7 +99,7 @@ headers_no_content_type(_) -> undefined. body(type) -> map(); body(nullable) -> false; -body(validate) -> [fun check_body/1]; +body(validator) -> [fun check_body/1]; body(_) -> undefined. request_timeout(type) -> non_neg_integer(); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 774d75157..327e356f2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -123,7 +123,7 @@ server_name_indication(_) -> undefined. verify_claims(type) -> list(); verify_claims(default) -> #{}; -verify_claims(validate) -> [fun check_verify_claims/1]; +verify_claims(validator) -> [fun do_check_verify_claims/1]; verify_claims(converter) -> fun(VerifyClaims) -> maps:to_list(VerifyClaims) @@ -298,12 +298,8 @@ do_verify_claims(Claims, [{Name, Value} | More]) -> {error, {claims, {Name, Value0}}} end. -check_verify_claims(Conf) -> - Claims = hocon_schema:get_value("verify_claims", Conf), - do_check_verify_claims(Claims). - do_check_verify_claims([]) -> - false; + true; do_check_verify_claims([{Name, Expected} | More]) -> check_claim_name(Name) andalso check_claim_expected(Expected) andalso diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 73412f388..0f8c23986 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -65,10 +65,10 @@ validations() -> base_url(type) -> url(); base_url(nullable) -> false; -base_url(validate) -> fun (#{query := _Query}) -> - {error, "There must be no query in the base_url"}; +base_url(validator) -> fun(#{query := _Query}) -> + {error, "There must be no query in the base_url"}; (_) -> ok - end; + end; base_url(_) -> undefined. connect_timeout(type) -> connect_timeout(); From bb4e5fdb4f8d49fb43420a69a104cceef9aa3d09 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 17 Sep 2021 10:26:18 +0800 Subject: [PATCH 66/84] chore(authn): fix dialyzer --- apps/emqx/src/emqx_authentication.erl | 61 +++++++++++++++----------- apps/emqx_authn/src/emqx_authn_api.erl | 12 +++-- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 8c682d1fc..bcb38471a 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -659,8 +659,12 @@ reply(Reply, State) -> certs_dir(Dirs) when is_list(Dirs) -> to_bin(filename:join([emqx:get_config([node, data_dir]), "certs/authn"] ++ Dirs)). -convert_certs(CertsDir, #{<<"ssl">> := SSLOpts} = Config) -> - NSSLOPts = lists:foldl(fun(K, Acc) -> +convert_certs(CertsDir, Config) -> + case maps:get(<<"ssl">>, Config, undefined) of + undefined -> + Config; + SSLOpts -> + NSSLOPts = lists:foldl(fun(K, Acc) -> case maps:get(K, Acc, undefined) of undefined -> Acc; PemBin -> @@ -669,32 +673,37 @@ convert_certs(CertsDir, #{<<"ssl">> := SSLOpts} = Config) -> Acc#{K => CertFile} end end, SSLOpts, [<<"certfile">>, <<"keyfile">>, <<"cacertfile">>]), - Config#{<<"ssl">> => NSSLOPts}; -convert_certs(_CertsDir, Config) -> - Config. + Config#{<<"ssl">> => NSSLOPts} + end. -convert_certs(CertsDir, #{<<"ssl">> := NewSSLOpts} = NewConfig, OldConfig) -> - OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), - Diff = diff_certs(NewSSLOpts, OldSSLOpts), - NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> - Acc#{K => maps:get(K, OldSSLOpts)}; - ({_, K}, Acc) -> - CertFile = generate_filename(CertsDir, K), - ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), - Acc#{K => CertFile} - end, NewSSLOpts, Diff), - NewConfig#{<<"ssl">> => NSSLOpts}; -convert_certs(_CertsDir, NewConfig, _OldConfig) -> - NewConfig. +convert_certs(CertsDir, NewConfig, OldConfig) -> + case maps:get(<<"ssl">>, NewConfig, undefined) of + undefined -> + NewConfig; + NewSSLOpts -> + OldSSLOpts = maps:get(<<"ssl">>, OldConfig, #{}), + Diff = diff_certs(NewSSLOpts, OldSSLOpts), + NSSLOpts = lists:foldl(fun({identical, K}, Acc) -> + Acc#{K => maps:get(K, OldSSLOpts)}; + ({_, K}, Acc) -> + CertFile = generate_filename(CertsDir, K), + ok = save_cert_to_file(CertFile, maps:get(K, NewSSLOpts)), + Acc#{K => CertFile} + end, NewSSLOpts, Diff), + NewConfig#{<<"ssl">> => NSSLOpts} + end. -clear_certs(CertsDir, #{<<"ssl">> := SSLOpts}) -> - lists:foreach( - fun({_, Filename}) -> - _ = file:delete(filename:join([CertsDir, Filename])) - end, - maps:to_list(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], SSLOpts))); -clear_certs(_CertsDir, _Config) -> - ok. +clear_certs(CertsDir, Config) -> + case maps:get(<<"ssl">>, Config, undefined) of + undefined -> + ok; + SSLOpts -> + lists:foreach( + fun({_, Filename}) -> + _ = file:delete(filename:join([CertsDir, Filename])) + end, + maps:to_list(maps:with([<<"certfile">>, <<"keyfile">>, <<"cacertfile">>], SSLOpts))) + end. save_cert_to_file(Filename, PemBin) -> case public_key:pem_decode(PemBin) =/= [] of diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 0eb2f5cfa..b58a2a214 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1840,11 +1840,10 @@ find_listener(ListenerID) -> {ok, {Type, Name}} end. -create_authenticator(ConfKeyPath, ChainName0, Config) -> - ChainName = to_atom(ChainName0), - case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of +create_authenticator(ConfKeyPath, ChainName, Config) -> + case update_config(ConfKeyPath, {create_authenticator, to_atom(ChainName), Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, - raw_config := AuthenticatorsConfig}} -> + raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; {error, {_, _, Reason}} -> @@ -1866,9 +1865,8 @@ list_authenticator(ConfKeyPath, AuthenticatorID) -> serialize_error(Reason) end. -update_authenticator(ConfKeyPath, ChainName0, AuthenticatorID, Config) -> - ChainName = to_atom(ChainName0), - case update_config(ConfKeyPath, {update_authenticator, ChainName, AuthenticatorID, Config}) of +update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> + case update_config(ConfKeyPath, {update_authenticator, to_atom(ChainName), AuthenticatorID, Config}) of {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), From 65633a4ad2c6ebd7cd6a7c6d50b41f4e19cbe2b8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 18 Sep 2021 12:34:20 +0800 Subject: [PATCH 67/84] chore(cluster-call): more detail about failed case --- apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl index cb07db37e..d39dc3c6e 100644 --- a/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_machine/test/emqx_cluster_rpc_SUITE.erl @@ -179,14 +179,23 @@ t_del_stale_mfa(_Config) -> t_skip_failed_commit(_Config) -> emqx_cluster_rpc:reset(), {atomic, []} = emqx_cluster_rpc:status(), - {ok, _, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000), + {ok, 1, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000), + {atomic, List1} = emqx_cluster_rpc:status(), + Node = node(), + ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], + tnx_ids(List1)), {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, {ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), ok = gen_server:call(?NODE2, skip_failed_commit, 5000), - {atomic, List} = emqx_cluster_rpc:status(), - ?assertEqual([1, 2, 2], lists:sort(lists:map(fun(#{tnx_id := TnxId}) -> TnxId end, List))), + {atomic, List2} = emqx_cluster_rpc:status(), + ?assertEqual([{Node, 2}, {{Node, ?NODE2}, 2}, {{Node, ?NODE3}, 1}], + tnx_ids(List2)), ok. +tnx_ids(Status) -> + lists:sort(lists:map(fun(#{tnx_id := TnxId, node := Node}) -> + {Node, TnxId} end, Status)). + start() -> {ok, Pid1} = emqx_cluster_rpc:start_link(), {ok, Pid2} = emqx_cluster_rpc:start_link({node(), ?NODE2}, ?NODE2, 500), From 2edd3879b87def3b326d5ce9fddcc0f1b8819100 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 19 Sep 2021 14:33:43 +0200 Subject: [PATCH 68/84] feat(config): save override config in hocon pretty-print format --- apps/emqx/src/emqx_config.erl | 4 ++-- apps/emqx/src/emqx_config_handler.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bd6e14e8e..98466d3df 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -355,7 +355,7 @@ save_to_override_conf(RawConf) -> undefined -> ok; FileName -> ok = filelib:ensure_dir(FileName), - case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of + case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of ok -> ok; {error, Reason} -> logger:error("write to ~s failed, ~p", [FileName, Reason]), @@ -424,7 +424,7 @@ root_names_from_conf(RawConf) -> [Name || Name <- get_root_names(), lists:member(Name, Keys)]. atom(Bin) when is_binary(Bin) -> - binary_to_existing_atom(Bin, latin1); + binary_to_existing_atom(Bin, utf8); atom(Str) when is_list(Str) -> list_to_existing_atom(Str); atom(Atom) when is_atom(Atom) -> diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index d92f1d35a..e47bb489e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -310,4 +310,4 @@ safe_atom(Bin) when is_binary(Bin) -> safe_atom(Str) when is_list(Str) -> list_to_existing_atom(Str); safe_atom(Atom) when is_atom(Atom) -> - Atom. \ No newline at end of file + Atom. From 9bf284d651b261f4751bc7df5ef3bf7513337246 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 20 Sep 2021 14:01:21 +0300 Subject: [PATCH 69/84] fix(README): fix command to run in build instructions --- README-CN.md | 2 +- README-RU.md | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README-CN.md b/README-CN.md index 80e926199..0631a845c 100644 --- a/README-CN.md +++ b/README-CN.md @@ -49,7 +49,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p git clone https://github.com/emqx/emqx.git cd emqx make -_build/emqx/rel/emqx/bin console +_build/emqx/rel/emqx/bin/emqx console ``` 对于 4.3 之前的版本，通过另外一个仓库构建： diff --git a/README-RU.md b/README-RU.md index e02f47aa4..6707ef939 100644 --- a/README-RU.md +++ b/README-RU.md @@ -50,7 +50,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p git clone https://github.com/emqx/emqx.git cd emqx make -_build/emqx/rel/emqx/bin console +_build/emqx/rel/emqx/bin/emqx console ``` Более ранние релизы могут быть собраны с помощью другого репозитория: diff --git a/README.md b/README.md index f60ed3cd9..dac48c2f2 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ For 4.3 and later versions. git clone https://github.com/emqx/emqx.git cd emqx make -_build/emqx/rel/emqx/bin console +_build/emqx/rel/emqx/bin/emqx console ``` For earlier versions, release has to be built from another repo. From dab5fbf28547346b0e6e56f70133ca91a4ab792e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 21 Sep 2021 21:41:16 +0800 Subject: [PATCH 70/84] feat(swagger): swagger support hocon schema --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_schema.erl | 24 +- apps/emqx_dashboard/rebar.config | 1 + .../emqx_dashboard/src/emqx_dashboard_api.erl | 224 ++++----- .../src/emqx_dashboard_schema.erl | 4 +- .../src/emqx_dashboard_swagger.erl | 327 ++++++++++++ .../test/emqx_swagger_parameter_SUITE.erl | 261 ++++++++++ .../test/emqx_swagger_remote_schema.erl | 60 +++ .../test/emqx_swagger_requestBody_SUITE.erl | 471 ++++++++++++++++++ .../test/emqx_swagger_response_SUITE.erl | 292 +++++++++++ rebar.config | 4 +- 11 files changed, 1532 insertions(+), 138 deletions(-) create mode 100644 apps/emqx_dashboard/src/emqx_dashboard_swagger.erl create mode 100644 apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl create mode 100644 apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl create mode 100644 apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl create mode 100644 apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7a43249a4..54735360b 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -16,7 +16,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d7caeb971..6c2245ad3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -156,11 +156,11 @@ fields("stats") -> fields("authorization") -> [ {"no_match", - sc(union(allow, deny), + sc(hoconsc:union([allow, deny]), #{ default => allow })} , {"deny_action", - sc(union(ignore, disconnect), + sc(hoconsc:union([ignore, disconnect]), #{ default => ignore })} , {"cache", @@ -939,9 +939,9 @@ ssl(Defaults) -> sc(boolean(), #{ default => Df("secure_renegotiate", true) , desc => """ -SSL parameter renegotiation is a feature that allows a client and a server -to renegotiate the parameters of the SSL connection on the fly. -RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, you drop support for the insecure renegotiation, prone to MitM attacks. """ }) @@ -950,13 +950,13 @@ you drop support for the insecure renegotiation, prone to MitM attacks. sc(boolean(), #{ default => Df("client_renegotiation", true) , desc => """ -In protocols that support client-initiated renegotiation, -the cost of resources of such an operation is higher for the server than the client. -This can act as a vector for denial of service attacks. -The SSL application already takes measures to counter-act such attempts, -but client-initiated renegotiation can be strictly disabled by setting this option to false. -The default value is true. Note that disabling renegotiation can result in -long-lived connections becoming unusable due to limits on +In protocols that support client-initiated renegotiation, +the cost of resources of such an operation is higher for the server than the client. +This can act as a vector for denial of service attacks. +The SSL application already takes measures to counter-act such attempts, +but client-initiated renegotiation can be strictly disabled by setting this option to false. +The default value is true. Note that disabling renegotiation can result in +long-lived connections becoming unusable due to limits on the number of messages the underlying cipher suite can encipher. """ }) diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index bdb491bcb..d0a1fbde4 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -13,3 +13,4 @@ {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. +{eunit_first_files, ["test/emqx_swagger_remote_schema.erl"]}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 68c737488..deb7dc74c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -29,135 +29,132 @@ -behaviour(minirest_api). -include("emqx_dashboard.hrl"). +-include_lib("typerefl/include/types.hrl"). +-import(hoconsc, [mk/2, ref/2, array/1, enum/1]). --import(emqx_mgmt_util, [ schema/1 - , object_schema/1 - , object_schema/2 - , object_array_schema/1 - , bad_request/0 - , properties/1 - ]). - --export([api_spec/0]). - --export([ login/2 - , logout/2 - , users/2 - , user/2 - , change_pwd/2 - ]). +-export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). +-export([login/2, logout/2, users/2, user/2, change_pwd/2]). -define(EMPTY(V), (V == undefined orelse V == <<>>)). - -define(ERROR_USERNAME_OR_PWD, 'ERROR_USERNAME_OR_PWD'). +namespace() -> "dashboard". + api_spec() -> - {[ login_api() - , logout_api() - , users_api() - , user_api() - , change_pwd_api() - ], - []}. + emqx_dashboard_swagger:spec(?MODULE). -login_api() -> - AuthProps = properties([{username, string, <<"Username">>}, - {password, string, <<"Password">>}]), +paths() -> ["/login", "/logout", "/users", + "/users/:username", "/users/:username/change_pwd"]. - TokenProps = properties([{token, string, <<"JWT Token">>}, - {license, object, [{edition, string, <<"License">>, [community, enterprise]}]}, - {version, string}]), - Metadata = #{ +schema("/login") -> + #{ + operationId => login, post => #{ - tags => [dashboard], + tags => [<<"dashboard">>], description => <<"Dashboard Auth">>, - 'requestBody' => object_schema(AuthProps), + summary => <<"Dashboard Auth">>, + requestBody => + [ + {username, mk(binary(), + #{desc => <<"The User for which to create the token.">>, + maxLength => 100, example => <<"admin">>})}, + {password, mk(binary(), + #{desc => "password", example => "public"})} + ], responses => #{ - <<"200">> => - object_schema(TokenProps, <<"Dashboard Auth successfully">>), - <<"401">> => unauthorized_request() + 200 => [ + {token, mk(string(), #{desc => <<"JWT Token">>})}, + {license, [{edition, + mk(enum([community, enterprise]), #{desc => <<"license">>, + example => "community"})}]}, + {version, mk(string(), #{desc => <<"version">>, example => <<"5.0.0">>})}], + 401 => [ + {code, mk(string(), #{example => 'ERROR_USERNAME_OR_PWD'})}, + {message, mk(string(), #{example => "Unauthorized"})}] }, security => [] - } - }, - {"/login", Metadata, login}. - -logout_api() -> - LogoutProps = properties([{username, string, <<"Username">>}]), - Metadata = #{ + }}; +schema("/logout") -> + #{ + operationId => logout, post => #{ - tags => [dashboard], - description => <<"Dashboard Auth">>, - 'requestBody' => object_schema(LogoutProps), + tags => [<<"dashboard">>], + description => <<"Dashboard User logout">>, + requestBody => [ + {username, mk(binary(), + #{desc => <<"The User for which to create the token.">>, + maxLength => 100, example => <<"admin">>})} + ], responses => #{ - <<"200">> => schema(<<"Dashboard Auth successfully">>) + 200 => <<"Dashboard logout successfully">> } } - }, - {"/logout", Metadata, logout}. - -users_api() -> - BaseProps = properties([{username, string, <<"Username">>}, - {password, string, <<"Password">>}, - {tag, string, <<"Tag">>}]), - Metadata = #{ + }; +schema("/users") -> + #{ + operationId => users, get => #{ - tags => [dashboard], + tags => [<<"dashboard">>], description => <<"Get dashboard users">>, responses => #{ - <<"200">> => object_array_schema(maps:without([password], BaseProps)) - } - }, - post => #{ - tags => [dashboard], - description => <<"Create dashboard users">>, - 'requestBody' => object_schema(BaseProps), - responses => #{ - <<"200">> => schema(<<"Create Users successfully">>), - <<"400">> => bad_request() + 200 => mk(array(ref(?MODULE, user)), + #{desc => "User lists"}) } } - }, - {"/users", Metadata, users}. + }; -user_api() -> - Metadata = #{ - delete => #{ - tags => [dashboard], - description => <<"Delete dashboard users">>, - parameters => parameters(), - responses => #{ - <<"200">> => schema(<<"Delete User successfully">>), - <<"400">> => bad_request() - } - }, +schema("/users/:username") -> + #{ + operationId => user, put => #{ - tags => [dashboard], + tags => [<<"dashboard">>], description => <<"Update dashboard users">>, - parameters => parameters(), - 'requestBody' => object_schema(properties([{tag, string, <<"Tag">>}])), + parameters => [{username, mk(binary(), + #{in => path, example => <<"admin">>})}], + requestBody => [{tag, mk(binary(), #{desc => <<"Tag">>})}], responses => #{ - <<"200">> => schema(<<"Update Users successfully">>), - <<"400">> => bad_request() - } - } - }, - {"/users/:username", Metadata, user}. - -change_pwd_api() -> - Metadata = #{ + 200 => <<"Update User successfully">>, + 400 => [{code, mk(string(), #{example => 'UPDATE_FAIL'})}, + {message, mk(string(), #{example => "Update Failed unknown"})}]}}, + delete => #{ + tags => [<<"dashboard">>], + description => <<"Delete dashboard users">>, + parameters => [{username, mk(binary(), + #{in => path, example => <<"admin">>})}], + responses => #{ + 200 => <<"Delete User successfully">>, + 400 => [ + {code, mk(string(), #{example => 'CANNOT_DELETE_ADMIN'})}, + {message, mk(string(), #{example => "CANNOT DELETE ADMIN"})}]}} + }; +schema("/users/:username/change_pwd") -> + #{ + operationId => change_pwd, put => #{ - tags => [dashboard], + tags => [<<"dashboard">>], description => <<"Update dashboard users password">>, - parameters => parameters(), - 'requestBody' => object_schema(properties([old_pwd, new_pwd])), + parameters => [{username, mk(binary(), + #{in => path, required => true, example => <<"admin">>})}], + requestBody => [ + {old_pwd, mk(binary(), #{required => true})}, + {new_pwd, mk(binary(), #{required => true})} + ], responses => #{ - <<"200">> => schema(<<"Update Users password successfully">>), - <<"400">> => bad_request() - } - } - }, - {"/users/:username/change_pwd", Metadata, change_pwd}. + 200 => <<"Update user password successfully">>, + 400 => [ + {code, mk(string(), #{example => 'UPDATE_FAIL'})}, + {message, mk(string(), #{example => "Failed Reason"})}]}} + }. + +fields(user) -> + [ + {tag, + mk(string(), + #{desc => <<"tag">>, example => "administrator"})}, + {username, + mk(string(), + #{desc => <<"username">>, example => "emqx"})} + ]. login(post, #{body := Params}) -> Username = maps:get(<<"username">>, Params), @@ -171,7 +168,7 @@ login(post, #{body := Params}) -> end. logout(_, #{body := #{<<"username">> := Username}, - headers := #{<<"authorization">> := <<"Bearer ", Token/binary>>}}) -> + headers := #{<<"authorization">> := <<"Bearer ", Token/binary>>}}) -> case emqx_dashboard_admin:destroy_token_by_username(Username, Token) of ok -> 200; @@ -187,9 +184,9 @@ users(post, #{body := Params}) -> Username = maps:get(<<"username">>, Params), Password = maps:get(<<"password">>, Params), case ?EMPTY(Username) orelse ?EMPTY(Password) of - true -> + true -> {400, #{code => <<"CREATE_USER_FAIL">>, - message => <<"Username or password undefined">>}}; + message => <<"Username or password undefined">>}}; false -> case emqx_dashboard_admin:add_user(Username, Password, Tag) of ok -> {200}; @@ -208,8 +205,8 @@ user(put, #{bindings := #{username := Username}, body := Params}) -> user(delete, #{bindings := #{username := Username}}) -> case Username == <<"admin">> of - true -> {400, #{code => <<"CONNOT_DELETE_ADMIN">>, - message => <<"Cannot delete admin">>}}; + true -> {400, #{code => <<"CANNOT_DELETE_ADMIN">>, + message => <<"Cannot delete admin">>}}; false -> _ = emqx_dashboard_admin:remove_user(Username), {200} @@ -226,20 +223,3 @@ change_pwd(put, #{bindings := #{username := Username}, body := Params}) -> row(#mqtt_admin{username = Username, tags = Tag}) -> #{username => Username, tag => Tag}. - -parameters() -> - [#{ - name => username, - in => path, - required => true, - schema => #{type => string}, - example => <<"admin">> - }]. - -unauthorized_request() -> - object_schema( - properties([{message, string}, - {code, string, <<"Resp Code">>, [?ERROR_USERNAME_OR_PWD]} - ]), - <<"Unauthorized">> - ). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 3ba3dc803..94cfaddad 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -18,8 +18,10 @@ -include_lib("typerefl/include/types.hrl"). -export([ roots/0 - , fields/1]). + , fields/1 + ,namespace/0]). +namespace() -> <<"dashboard">>. roots() -> ["emqx_dashboard"]. fields("emqx_dashboard") -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl new file mode 100644 index 000000000..517fe5bb5 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -0,0 +1,327 @@ +-module(emqx_dashboard_swagger). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +%% API +-export([spec/1]). +-export([translate_req/2]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +-define(METHODS, [get, post, put, head, delete, patch, options, trace]). + +-define(DEFAULT_FIELDS, [example, allowReserved, style, + explode, maxLength, allowEmptyValue, deprecated, minimum, maximum]). + +-define(DEFAULT_FILTER, #{filter => fun ?MODULE:translate_req/2}). + +-define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}). + +-define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). +-define(TO_COMPONENTS(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])). + +-spec(spec(module()) -> {list({Path, Specs, OperationId, Options}), list(Component)} when + Path :: string()|binary(), + Specs :: map(), + OperationId :: atom(), + Options :: #{filter => fun((map(), + #{module => module(), path => string(), method => atom()}) -> map())}, + Component :: map()). +spec(Module) -> + Paths = apply(Module, paths, []), + {ApiSpec, AllRefs} = + lists:foldl(fun(Path, {AllAcc, AllRefsAcc}) -> + {OperationId, Specs, Refs} = parse_spec_ref(Module, Path), + {[{Path, Specs, OperationId, ?DEFAULT_FILTER} | AllAcc], + Refs ++ AllRefsAcc} + end, {[], []}, Paths), + {ApiSpec, components(lists:usort(AllRefs))}. + +-spec(translate_req(#{binding => list(), query_string => list(), body => map()}, + #{module => module(), path => string(), method => atom()}) -> + {ok, #{binding => list(), query_string => list(), body => map()}}| + {400, 'BAD_REQUEST', binary()}). +translate_req(Request, #{module := Module, path := Path, method := Method}) -> + #{Method := Spec} = apply(Module, schema, [Path]), + try + Params = maps:get(parameters, Spec, []), + Body = maps:get(requestBody, Spec, []), + {Bindings, QueryStr} = check_parameters(Request, Params), + NewBody = check_requestBody(Request, Body, Module, hoconsc:is_schema(Body)), + {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}} + catch throw:Error -> + {_, [{validation_error, ValidErr}]} = Error, + #{path := Key, reason := Reason} = ValidErr, + {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~s : ~p", [Key, Reason]))} + end. + +parse_spec_ref(Module, Path) -> + Schema = + try + erlang:apply(Module, schema, [Path]) + catch error: Reason -> %% better error message + throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}}) + end, + {Specs, Refs} = maps:fold(fun(Method, Meta, {Acc, RefsAcc}) -> + (not lists:member(Method, ?METHODS)) + andalso throw({error, #{module => Module, path => Path, method => Method}}), + {Spec, SubRefs} = meta_to_spec(Meta, Module), + {Acc#{Method => Spec}, SubRefs ++ RefsAcc} + end, {#{}, []}, + maps:without([operationId], Schema)), + {maps:get(operationId, Schema), Specs, Refs}. + +check_parameters(Request, Spec) -> + #{bindings := Bindings, query_string := QueryStr} = Request, + BindingsBin = maps:fold(fun(Key, Value, Acc) -> Acc#{atom_to_binary(Key) => Value} end, #{}, Bindings), + check_parameter(Spec, BindingsBin, QueryStr, #{}, #{}). + +check_parameter([], _Bindings, _QueryStr, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr}; +check_parameter([{Name, Type} | Spec], Bindings, QueryStr, BindingsAcc, QueryStrAcc) -> + Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, + case hocon_schema:field_schema(Type, in) of + path -> + NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true}), + NewBindingsAcc = maps:merge(BindingsAcc, NewBindings), + check_parameter(Spec, Bindings, QueryStr, NewBindingsAcc, QueryStrAcc); + query -> + NewQueryStr = hocon_schema:check_plain(Schema, QueryStr), + NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), + check_parameter(Spec, Bindings, QueryStr, BindingsAcc, NewQueryStrAcc) + end. + +check_requestBody(#{body := Body}, Schema, Module, true) -> + Type0 = hocon_schema:field_schema(Schema, type), + Type = + case Type0 of + ?REF(StructName) -> ?R_REF(Module, StructName); + _ -> Type0 + end, + NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]}, + #{<<"root">> := NewBody} = hocon_schema:check_plain(NewSchema, #{<<"root">> => Body}), + NewBody; +%% TODO not support nest object check yet, please use ref! +%% RequestBody = [ {per_page, mk(integer(), #{}}, +%% {nest_object, [ +%% {good_nest_1, mk(integer(), #{})}, +%% {good_nest_2, mk(ref(?MODULE, good_ref), #{})} +%% ]} +%% ] +check_requestBody(#{body := Body}, Spec, _Module, false) -> + lists:foldl(fun({Name, Type}, Acc) -> + Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, + maps:merge(Acc, hocon_schema:check_plain(Schema, Body)) + end, #{}, Spec). + +%% tags, description, summary, security, deprecated +meta_to_spec(Meta, Module) -> + {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module), + {RequestBody, Refs2} = requestBody(maps:get(requestBody, Meta, []), Module), + {Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module), + { + to_spec(Meta, Params, RequestBody, Responses), + lists:usort(Refs1 ++ Refs2 ++ Refs3) + }. + +to_spec(Meta, Params, [], Responses) -> + Spec = maps:without([parameters, requestBody, responses], Meta), + Spec#{parameters => Params, responses => Responses}; +to_spec(Meta, Params, RequestBody, Responses) -> + Spec = to_spec(Meta, Params, [], Responses), + maps:put(requestBody, RequestBody, Spec). + +parameters(Params, Module) -> + {SpecList, AllRefs} = + lists:foldl(fun({Name, Type}, {Acc, RefsAcc}) -> + In = hocon_schema:field_schema(Type, in), + In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}), + Nullable = hocon_schema:field_schema(Type, nullable), + Default = hocon_schema:field_schema(Type, default), + HoconType = hocon_schema:field_schema(Type, type), + Meta = init_meta(Nullable, Default), + {ParamType, Refs} = hocon_schema_to_spec(HoconType, Module), + Spec0 = init_prop([required | ?DEFAULT_FIELDS], + #{schema => maps:merge(ParamType, Meta), name => Name, in => In}, Type), + Spec1 = trans_required(Spec0, Nullable, In), + Spec2 = trans_desc(Spec1, Type), + {[Spec2 | Acc], Refs ++ RefsAcc} + end, {[], []}, Params), + {lists:reverse(SpecList), AllRefs}. + +init_meta(Nullable, Default) -> + Init = + case Nullable of + true -> #{nullable => true}; + _ -> #{} + end, + case Default =:= undefined of + true -> Init; + false -> Init#{default => Default} + end. + +init_prop(Keys, Init, Type) -> + lists:foldl(fun(Key, Acc) -> + case hocon_schema:field_schema(Type, Key) of + undefined -> Acc; + Schema -> Acc#{Key => to_bin(Schema)} + end + end, Init, Keys). + +trans_required(Spec, false, _) -> Spec#{required => true}; +trans_required(Spec, _, path) -> Spec#{required => true}; +trans_required(Spec, _, _) -> Spec. + +trans_desc(Spec, Hocon) -> + case hocon_schema:field_schema(Hocon, desc) of + undefined -> Spec; + Desc -> Spec#{description => Desc} + end. + +requestBody([], _Module) -> {[], []}; +requestBody(Schema, Module) -> + {Props, Refs} = + case hoconsc:is_schema(Schema) of + true -> + HoconSchema = hocon_schema:field_schema(Schema, type), + hocon_schema_to_spec(HoconSchema, Module); + false -> parse_object(Schema, Module) + end, + {#{<<"content">> => #{<<"application/json">> => #{<<"schema">> => Props}}}, + Refs}. + +responses(Responses, Module) -> + {Spec, Refs, _} = maps:fold(fun response/3, {#{}, [], Module}, Responses), + {Spec, Refs}. + +response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) -> + {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module}; +response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) -> + response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module}); +response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) -> + {Spec, Refs} = hocon_schema_to_spec(RRef, Module), + Content = #{<<"application/json">> => #{<<"schema">> => Spec}}, + {Acc#{integer_to_binary(Status) => #{<<"content">> => Content}}, Refs ++ RefsAcc, Module}; +response(Status, Schema, {Acc, RefsAcc, Module}) -> + case hoconsc:is_schema(Schema) of + true -> + Hocon = hocon_schema:field_schema(Schema, type), + {Spec, Refs} = hocon_schema_to_spec(Hocon, Module), + Init = trans_desc(#{}, Schema), + Content = #{<<"application/json">> => #{<<"schema">> => Spec}}, + {Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, Refs ++ RefsAcc, Module}; + false -> + {Props, Refs} = parse_object(Schema, Module), + Content = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => Props}}}, + {Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module} + end. + +components(Refs) -> + lists:sort(maps:fold(fun(K, V, Acc) -> [#{K => V} | Acc] end, [], + components(Refs, #{}, []))). + +components([], SpecAcc, []) -> SpecAcc; +components([], SpecAcc, SubRefAcc) -> components(SubRefAcc, SpecAcc, []); +components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> + Props = apply(Module, fields, [Field]), + Namespace = namespace(Module), + {Object, SubRefs} = parse_object(Props, Module), + NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object}, + components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). + +namespace(Module) -> + case hocon_schema:namespace(Module) of + undefined -> Module; + NameSpace -> NameSpace + end. + +hocon_schema_to_spec(?R_REF(Module, StructName), _LocalModule) -> + {#{<<"$ref">> => ?TO_COMPONENTS(Module, StructName)}, + [{Module, StructName}]}; +hocon_schema_to_spec(?REF(StructName), LocalModule) -> + {#{<<"$ref">> => ?TO_COMPONENTS(LocalModule, StructName)}, + [{LocalModule, StructName}]}; +hocon_schema_to_spec(Type, _LocalModule) when ?IS_TYPEREFL(Type) -> + {typename_to_spec(typerefl:name(Type)), []}; +hocon_schema_to_spec(?ARRAY(Item), LocalModule) -> + {Schema, Refs} = hocon_schema_to_spec(Item, LocalModule), + {#{type => array, items => Schema}, Refs}; +hocon_schema_to_spec(?ENUM(Items), _LocalModule) -> + {#{type => string, enum => Items}, []}; +hocon_schema_to_spec(?UNION(Types), LocalModule) -> + {OneOf, Refs} = lists:foldl(fun(Type, {Acc, RefsAcc}) -> + {Schema, SubRefs} = hocon_schema_to_spec(Type, LocalModule), + {[Schema | Acc], SubRefs ++ RefsAcc} + end, {[], []}, Types), + {#{<<"oneOf">> => OneOf}, Refs}; +hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> + {#{type => string, enum => [Atom]}, []}. + +typename_to_spec("boolean()") -> #{type => boolean, example => true}; +typename_to_spec("binary()") -> #{type => string, example =><<"binary example">>}; +typename_to_spec("float()") -> #{type =>number, example =>3.14159}; +typename_to_spec("integer()") -> #{type =>integer, example =>100}; +typename_to_spec("number()") -> #{type =>number, example =>42}; +typename_to_spec("string()") -> #{type =>string, example =><<"string example">>}; +typename_to_spec("atom()") -> #{type =>string, example =>atom}; +typename_to_spec("duration()") -> #{type =>string, example =><<"12m">>}; +typename_to_spec("duration_s()") -> #{type =>string, example =><<"1h">>}; +typename_to_spec("duration_ms()") -> #{type =>string, example =><<"32s">>}; +typename_to_spec("percent()") -> #{type =>number, example =><<"12%">>}; +typename_to_spec("file()") -> #{type =>string, example =><<"/path/to/file">>}; +typename_to_spec("ip_port()") -> #{type => string, example =><<"127.0.0.1:80">>}; +typename_to_spec(Name) -> + case string:split(Name, "..") of + [MinStr, MaxStr] -> %% 1..10 + {Min, []} = string:to_integer(MinStr), + {Max, []} = string:to_integer(MaxStr), + #{type => integer, example => Min, minimum => Min, maximum => Max}; + _ -> %% Module:Type(). + case string:split(Name, ":") of + [_Module, Type] -> typename_to_spec(Type); + _ -> throw({error, #{msg => <<"Unsupport Type">>, type => Name}}) + end + end. + +to_bin(List) when is_list(List) -> list_to_binary(List); +to_bin(B) when is_boolean(B) -> B; +to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +to_bin(X) -> X. + +parse_object(PropList = [_|_], Module) when is_list(PropList) -> + {Props, Required, Refs} = + lists:foldl(fun({Name, Hocon}, {Acc, RequiredAcc, RefsAcc}) -> + NameBin = to_bin(Name), + case hoconsc:is_schema(Hocon) of + true -> + HoconType = hocon_schema:field_schema(Hocon, type), + Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), + Init = trans_desc(Init0, Hocon), + {Prop, Refs1} = hocon_schema_to_spec(HoconType, Module), + NewRequiredAcc = + case is_required(Hocon) of + true -> [NameBin | RequiredAcc]; + false -> RequiredAcc + end, + {[{NameBin, maps:merge(Prop, Init)} | Acc], NewRequiredAcc, Refs1 ++ RefsAcc}; + false -> + {SubObject, SubRefs} = parse_object(Hocon, Module), + {[{NameBin, SubObject} | Acc], RequiredAcc, SubRefs ++ RefsAcc} + end + end, {[], [], []}, PropList), + Object = #{<<"type">> => object, <<"properties">> => lists:reverse(Props)}, + case Required of + [] -> {Object, Refs}; + _ -> {maps:put(required, Required, Object), Refs} + end; +parse_object(Other, Module) -> + erlang:throw({error, + #{msg => <<"Object only supports not empty proplists">>, + args => Other, module => Module}}). + +is_required(Hocon) -> + hocon_schema:field_schema(Hocon, required) =:= true orelse + hocon_schema:field_schema(Hocon, nullable) =:= false. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl new file mode 100644 index 000000000..74cb53a26 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -0,0 +1,261 @@ +-module(emqx_swagger_parameter_SUITE). +-behaviour(minirest_api). +-behaviour(hocon_schema). + +%% API +-export([paths/0, api_spec/0, schema/1]). +-export([t_in_path/1, t_in_query/1, t_in_mix/1, t_without_in/1]). +-export([t_require/1, t_nullable/1, t_method/1, t_api_spec/1]). +-export([t_in_path_trans/1, t_in_query_trans/1, t_in_mix_trans/1]). +-export([t_in_path_trans_error/1, t_in_query_trans_error/1, t_in_mix_trans_error/1]). +-export([all/0, suite/0, groups/0]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-import(hoconsc, [mk/2]). + +-define(METHODS, [get, post, put, head, delete, patch, options, trace]). + +all() -> [{group, spec}, {group, validation}]. +suite() -> [{timetrap, {minutes, 1}}]. +groups() -> [ + {spec, [parallel], [t_api_spec, t_in_path, t_in_query, t_in_mix, + t_without_in, t_require, t_nullable, t_method]}, + {validation, [parallel], [t_in_path_trans, t_in_query_trans, t_in_mix_trans, + t_in_path_trans_error, t_in_query_trans_error, t_in_mix_trans_error]} +]. + +t_in_path(_Config) -> + Expect = + [#{description => <<"Indicates which sorts of issues to return">>, + example => <<"all">>, in => path, name => filter, + required => true, + schema => #{enum => [assigned, created, mentioned, all], type => string}} + ], + validate("/test/in/:filter", Expect), + ok. + +t_in_query(_Config) -> + Expect = + [#{description => <<"results per page (max 100)">>, + example => 1, in => query, name => per_page, + schema => #{example => 1, maximum => 100, minimum => 1, type => integer}}], + validate("/test/in/query", Expect), + ok. + +t_in_mix(_Config) -> + Expect = + [#{description => <<"Indicates which sorts of issues to return">>, + example => <<"all">>,in => query,name => filter, + schema => #{enum => [assigned,created,mentioned,all],type => string}}, + #{description => <<"Indicates the state of the issues to return.">>, + example => <<"12m">>,in => path,name => state,required => true, + schema => #{example => <<"1h">>,type => string}}, + #{example => 10,in => query,name => per_page, required => false, + schema => #{default => 5,example => 1,maximum => 50,minimum => 1, type => integer}}, + #{in => query,name => is_admin, schema => #{example => true,type => boolean}}, + #{in => query,name => timeout, + schema => #{<<"oneOf">> => [#{enum => [infinity],type => string}, + #{example => 30,maximum => 60,minimum => 30, type => integer}]}}], + ExpectMeta = #{ + tags => [tags, good], + description => <<"good description">>, + summary => <<"good summary">>, + security => [], + deprecated => true, + responses => #{<<"200">> => #{description => <<"ok">>}}}, + GotSpec = validate("/test/in/mix/:state", Expect), + ?assertEqual(ExpectMeta, maps:without([parameters], maps:get(post, GotSpec))), + ok. + +t_without_in(_Config) -> + ?assertThrow({error, <<"missing in:path/query field in parameters">>}, + emqx_dashboard_swagger:parse_spec_ref(?MODULE, "/test/without/in")), + ok. + +t_require(_Config) -> + ExpectSpec = [#{ + in => query,name => userid, required => false, + schema => #{example => <<"binary example">>, type => string}}], + validate("/required/false", ExpectSpec), + ok. + +t_nullable(_Config) -> + NullableFalse = [#{in => query,name => userid, required => true, + schema => #{example => <<"binary example">>, type => string}}], + NullableTrue = [#{in => query,name => userid, + schema => #{example => <<"binary example">>, type => string, + nullable => true}}], + validate("/nullable/false", NullableFalse), + validate("/nullable/true", NullableTrue), + ok. + +t_method(_Config) -> + PathOk = "/method/ok", + PathError = "/method/error", + {test, Spec, []} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathOk), + ?assertEqual(lists:sort(?METHODS), lists:sort(maps:keys(Spec))), + ?assertThrow({error, #{module := ?MODULE, path := PathError, method := bar}}, + emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathError)), + ok. + +t_in_path_trans(_Config) -> + Path = "/test/in/:filter", + Bindings = #{filter => <<"created">>}, + Expect = {ok,#{bindings => #{filter => created}, + body => #{}, query_string => #{}}}, + ?assertEqual(Expect, trans_parameters(Path, Bindings, #{})), + ok. + +t_in_query_trans(_Config) -> + Path = "/test/in/query", + Expect = {ok, #{bindings => #{},body => #{}, + query_string => #{<<"per_page">> => 100}}}, + ?assertEqual(Expect, trans_parameters(Path, #{}, #{<<"per_page">> => 100})), + ok. + +t_in_mix_trans(_Config) -> + Path = "/test/in/mix/:state", + Bindings = #{ + state => <<"12m">>, + per_page => <<"1">> + }, + Query = #{ + <<"filter">> => <<"created">>, + <<"is_admin">> => true, + <<"timeout">> => <<"34">> + }, + Expect = {ok, + #{body => #{}, + bindings => #{state => 720}, + query_string => #{<<"filter">> => created,<<"is_admin">> => true, <<"per_page">> => 5,<<"timeout">> => 34}}}, + ?assertEqual(Expect, trans_parameters(Path, Bindings, Query)), + ok. + +t_in_path_trans_error(_Config) -> + Path = "/test/in/:filter", + Bindings = #{filter => <<"created1">>}, + Expect = {400,'BAD_REQUEST', <<"filter : unable_to_convert_to_enum_symbol">>}, + ?assertEqual(Expect, trans_parameters(Path, Bindings, #{})), + ok. + +t_in_query_trans_error(_Config) -> + Path = "/test/in/query", + {400,'BAD_REQUEST', Reason} = trans_parameters(Path, #{}, #{<<"per_page">> => 101}), + ?assertNotEqual(nomatch, binary:match(Reason, [<<"per_page">>])), + ok. + +t_in_mix_trans_error(_Config) -> + Path = "/test/in/mix/:state", + Bindings = #{ + state => <<"1d2m">>, + per_page => <<"1">> + }, + Query = #{ + <<"filter">> => <<"cdreated">>, + <<"is_admin">> => true, + <<"timeout">> => <<"34">> + }, + Expect = {400,'BAD_REQUEST', <<"filter : unable_to_convert_to_enum_symbol">>}, + ?assertEqual(Expect, trans_parameters(Path, Bindings, Query)), + ok. + +t_api_spec(_Config) -> + emqx_dashboard_swagger:spec(?MODULE), + ok. + +validate(Path, ExpectParams) -> + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Params = maps:get(parameters, maps:get(post, Spec)), + ?assertEqual(ExpectParams, Params), + ?assertEqual([], Refs), + Spec. + +trans_parameters(Path, Bindings, QueryStr) -> + Meta = #{module => ?MODULE, method => post, path => Path}, + Request = #{bindings => Bindings, query_string => QueryStr, body => #{}}, + emqx_dashboard_swagger:translate_req(Request, Meta). + +api_spec() -> emqx_dashboard_swagger:spec(?MODULE). + +paths() -> ["/test/in/:filter", "/test/in/query", "/test/in/mix/:state", + "/required/false", "/nullable/false", "/nullable/true", "/method/ok"]. + +schema("/test/in/:filter") -> + #{ + operationId => test, + post => #{ + parameters => [ + {filter, + mk(hoconsc:enum([assigned, created, mentioned, all]), + #{in => path, desc => <<"Indicates which sorts of issues to return">>, example => "all"})} + ], + responses => #{200 => <<"ok">>} + } + }; +schema("/test/in/query") -> + #{ + operationId => test, + post => #{ + parameters => [ + {per_page, + mk(range(1, 100), + #{in => query, desc => <<"results per page (max 100)">>, example => 1})} + ], + responses => #{200 => <<"ok">>} + } + }; +schema("/test/in/mix/:state") -> + #{ + operationId => test, + post => #{ + tags => [tags, good], + description => <<"good description">>, + summary => <<"good summary">>, + security => [], + deprecated => true, + parameters => [ + {filter, hoconsc:mk(hoconsc:enum([assigned, created, mentioned, all]), + #{in => query, desc => <<"Indicates which sorts of issues to return">>, example => "all"})}, + {state, mk(emqx_schema:duration_s(), + #{in => path, required => true, example => "12m", desc => <<"Indicates the state of the issues to return.">>})}, + {per_page, mk(range(1, 50), + #{in => query, required => false, example => 10, default => 5})}, + {is_admin, mk(boolean(), #{in => query})}, + {timeout, mk(hoconsc:union([range(30, 60), infinity]), #{in => query})} + ], + responses => #{200 => <<"ok">>} + } + }; +schema("/test/without/in") -> + #{ + operationId => test, + post => #{ + parameters => [ + {'x-request-id', mk(binary(), #{})} + ], + responses => #{200 => <<"ok">>} + } + }; +schema("/required/false") -> + to_schema([{'userid', mk(binary(), #{in => query, required => false})}]); +schema("/nullable/false") -> + to_schema([{'userid', mk(binary(), #{in => query, nullable => false})}]); +schema("/nullable/true") -> + to_schema([{'userid', mk(binary(), #{in => query, nullable => true})}]); +schema("/method/ok") -> + Response = #{responses => #{200 => <<"ok">>}}, + lists:foldl(fun(Method, Acc) -> Acc#{Method => Response} end, + #{operationId => test}, ?METHODS); +schema("/method/error") -> + #{operationId => test, bar => #{200 => <<"ok">>}}. +to_schema(Params) -> + #{ + operationId => test, + post => #{ + parameters => Params, + responses => #{200 => <<"ok">>} + } + }. diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl new file mode 100644 index 000000000..a8602a79e --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl @@ -0,0 +1,60 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_swagger_remote_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([ roots/0, fields/1,namespace/0]). +-import(hoconsc, [mk/2]). +namespace() -> <<"remote">>. +roots() -> ["root"]. + +fields("root") -> + [ + {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "ref1"), + hoconsc:ref(?MODULE, "ref2")]))}, + {default_username, fun default_username/1}, + {default_password, fun default_password/1}, + {sample_interval, mk(emqx_schema:duration_s(), #{default => "10s"})}, + {token_expired_time, mk(emqx_schema:duration(), #{default => "30m"})} + ]; + +fields("ref1") -> + [ + {"protocol", hoconsc:enum([http, https])}, + {"port", mk(integer(), #{default => 18083})} + ]; + +fields("ref2") -> + [ + {page, mk(range(1,100), #{desc => <<"good page">>})}, + {another_ref, hoconsc:ref(?MODULE, "ref3")} + ]; +fields("ref3") -> + [ + {ip, mk(emqx_schema:ip_port(), #{desc => <<"IP:Port">>, example => "127.0.0.1:80"})}, + {version, mk(string(), #{desc => "a good version", example => "1.0.0"})} + ]. + +default_username(type) -> string(); +default_username(default) -> "admin"; +default_username(nullable) -> false; +default_username(_) -> undefined. + +default_password(type) -> string(); +default_password(default) -> "public"; +default_password(nullable) -> false; +default_password(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl new file mode 100644 index 000000000..462b20fd4 --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -0,0 +1,471 @@ +-module(emqx_swagger_requestBody_SUITE). + +-behaviour(minirest_api). +-behaviour(hocon_schema). + +%% API +-export([paths/0, api_spec/0, schema/1, fields/1]). +-export([t_object/1, t_nest_object/1, t_api_spec/1, + t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, + t_ref_array_with_key/1, t_ref_array_without_key/1 +]). +-export([ + t_object_trans/1, t_nest_object_trans/1, t_local_ref_trans/1, + t_remote_ref_trans/1, t_nest_ref_trans/1, + t_ref_array_with_key_trans/1, t_ref_array_without_key_trans/1, + t_ref_trans_error/1, t_object_trans_error/1 +]). +-export([all/0, suite/0, groups/0]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-import(hoconsc, [mk/2]). + +all() -> [{group, spec}, {group, validation}]. + +suite() -> [{timetrap, {minutes, 1}}]. +groups() -> [ + {spec, [parallel], [ + t_api_spec, t_object, t_nest_object, + t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, + t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]}, + {validation, [parallel], + [ + t_object_trans, t_local_ref_trans, t_remote_ref_trans, + t_ref_array_with_key_trans, t_ref_array_without_key_trans, t_nest_ref_trans, + t_ref_trans_error, t_object_trans_error + %% t_nest_object_trans, + ]} +]. + +t_object(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => + #{<<"application/json">> => + #{<<"schema">> => + #{required => [<<"timeout">>, <<"per_page">>], + <<"properties">> =>[ + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + <<"type">> => object}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, good_ref}], + validate("/object", Spec, Refs), + ok. + +t_nest_object(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => + #{required => [<<"timeout">>], + <<"properties">> => + [{<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"nest_object">>, + #{<<"properties">> => + [{<<"good_nest_1">>, #{example => 100, type => integer}}, + {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],<<"type">> => object}}, + {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + <<"type">> => object}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, good_ref}], + validate("/nest/object", Spec, Refs), + ok. + +t_local_ref(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, good_ref}], + validate("/ref/local", Spec, Refs), + ok. + +t_remote_ref(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/remote.ref2">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{emqx_swagger_remote_schema, "ref2"}], + {_, Components} = validate("/ref/remote", Spec, Refs), + ExpectComponents = [ + #{<<"remote.ref2">> => #{<<"properties">> => [ + {<<"page">>, #{description => <<"good page">>,example => 1, maximum => 100,minimum => 1,type => integer}}, + {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/remote.ref3">>}}], <<"type">> => object}}, + #{<<"remote.ref3">> => #{<<"properties">> => [ + {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"version">>, #{description => "a good version", example => <<"1.0.0">>,type => string}}], + <<"type">> => object}}], + ?assertEqual(ExpectComponents, Components), + ok. + +t_nest_ref(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.nest_ref">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, nest_ref}], + ExpectComponents = lists:sort([ + #{<<"emqx_swagger_requestBody_SUITE.nest_ref">> => #{<<"properties">> => [ + {<<"env">>, #{enum => [test,dev,prod],type => string}}, + {<<"another_ref">>, #{description => "nest ref", <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + <<"type">> => object}}, + #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [ + {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}}, + {<<"tag">>, #{description => <<"tag">>, example => <<"binary example">>,type => string}}], + <<"type">> => object}}]), + {_, Components} = validate("/ref/nest/ref", Spec, Refs), + ?assertEqual(ExpectComponents, Components), + ok. + +t_none_ref(_Config) -> + Path = "/ref/none", + ?assertThrow({error, #{mfa := {?MODULE, schema, [Path]}}}, + emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path)), + ok. + +t_bad_ref(_Config) -> + Path = "/ref/bad", + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => + #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.bad_ref">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, bad_ref}], + Fields = fields(bad_ref), + ?assertThrow({error, #{msg := <<"Object only supports not empty proplists">>, args := Fields}}, + validate(Path, Spec, Refs)), + ok. + +t_ref_array_with_key(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{required => [<<"timeout">>], + <<"type">> => object, <<"properties">> => + [ + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}} + ]}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, good_ref}], + validate("/ref/array/with/key", Spec, Refs), + ok. + +t_ref_array_without_key(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => + #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, good_ref}], + validate("/ref/array/without/key", Spec, Refs), + ok. + +t_api_spec(_Config) -> + emqx_dashboard_swagger:spec(?MODULE), + ok. + +t_object_trans(_Config) -> + Path = "/object", + Body = #{ + <<"per_page">> => 1, + <<"timeout">> => <<"infinity">>, + <<"inner_ref">> => #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + } + }, + Expect = + #{ + bindings => #{}, + query_string => #{}, + body => + #{ + <<"per_page">> => 1, + <<"timeout">> => infinity, + <<"inner_ref">> => #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"god_tag">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 80}} + } + }, + {ok, ActualBody} = trans_requestBody(Path, Body), + ?assertEqual(Expect, ActualBody), + ok. + +t_nest_object_trans(_Config) -> + Path = "/nest/object", + Body = #{ + <<"timeout">> => "10m", + <<"per_page">> => 10, + <<"inner_ref">> => #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + }, + <<"nest_object">> => #{ + <<"good_nest_1">> => 1, + <<"good_nest_2">> => #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + } + } + }, + Expect = #{ + bindings => #{}, + query_string => #{}, + body => #{<<"per_page">> => 10, + <<"timeout">> => 600} + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. + +t_local_ref_trans(_Config) -> + Path = "/ref/local", + Body = #{ + <<"webhook-host">> => <<"127.0.0.1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"A">> + }, + Expect = #{ + bindings => #{}, + query_string => #{}, + body => #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"A">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 80} + } + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. + +t_remote_ref_trans(_Config) -> + Path = "/ref/remote", + Body = #{ + <<"page">> => 10, + <<"another_ref">> => #{ + <<"version">> => "2.1.0", + <<"ip">> => <<"198.12.2.1:89">>} + }, + Expect = #{ + bindings => #{}, + query_string => #{}, + body => #{ + <<"page">> => 10, + <<"another_ref">> => #{ + <<"version">> => "2.1.0", + <<"ip">> => {{198,12,2,1}, 89}} + } + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. + +t_nest_ref_trans(_Config) -> + Path = "/ref/nest/ref", + Body = #{<<"env">> => <<"prod">>, + <<"another_ref">> => #{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => "127.0.0.1:80" + }}, + Expect = #{ + bindings => #{}, + query_string => #{}, + body => #{ + <<"another_ref">> => #{ + <<"log_dir">> => "var/log/dev", <<"tag">> => <<"A">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 80}}, + <<"env">> => prod} + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. + +t_ref_array_with_key_trans(_Config) -> + Path = "/ref/array/with/key", + Body = #{ + <<"per_page">> => 100, + <<"timeout">> => "100m", + <<"array_refs">> => [ + #{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => "127.0.0.1:80" + }, + #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"B">>, + <<"webhook-host">> => "127.0.0.1:81" + }] + }, + Expect = #{ + bindings => #{}, + query_string => #{}, + body => #{ + <<"per_page">> => 100, + <<"timeout">> => 6000, + <<"array_refs">> => [ + #{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 80} + }, + #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"B">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 81} + } + ] + } + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. +t_ref_array_without_key_trans(_Config) -> + Path = "/ref/array/without/key", + Body = [#{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => "127.0.0.1:80" + }, + #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"B">>, + <<"webhook-host">> => "127.0.0.1:81" + }], + Expect = #{ + bindings => #{}, + query_string => #{}, + body => [ + #{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 80} + }, + #{ + <<"log_dir">> => "var/log/test", + <<"tag">> => <<"B">>, + <<"webhook-host">> => {{127, 0, 0, 1}, 81} + }] + }, + {ok, NewRequest} = trans_requestBody(Path, Body), + ?assertEqual(Expect, NewRequest), + ok. + +t_ref_trans_error(_Config) -> + Path = "/ref/nest/ref", + Body = #{<<"env">> => <<"prod">>, + <<"another_ref">> => #{ + <<"log_dir">> => "var/log/dev", + <<"tag">> => <<"A">>, + <<"webhook-host">> => "127.0..0.1:80" + }}, + {400, 'BAD_REQUEST', _} = trans_requestBody(Path, Body), + ok. + +t_object_trans_error(_Config) -> + Path = "/object", + Body = #{ + <<"per_page">> => 99, + <<"timeout">> => <<"infinity">>, + <<"inner_ref">> => #{ + <<"webhook-host">> => <<"127.0.0..1:80">>, + <<"log_dir">> => <<"var/log/test">>, + <<"tag">> => <<"god_tag">> + } + }, + {400, 'BAD_REQUEST', Reason} = trans_requestBody(Path, Body), + ?assertNotEqual(nomatch, binary:match(Reason, [<<"webhook-host">>])), + ok. + +validate(Path, ExpectSpec, ExpectRefs) -> + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + ?assertEqual(ExpectSpec, Spec), + ?assertEqual(ExpectRefs, Refs), + {Spec, emqx_dashboard_swagger:components(Refs)}. + +trans_requestBody(Path, Body) -> + Meta = #{module => ?MODULE, method => post, path => Path}, + Request = #{bindings => #{}, query_string => #{}, body => Body}, + emqx_dashboard_swagger:translate_req(Request, Meta). + +api_spec() -> emqx_dashboard_swagger:spec(?MODULE). +paths() -> + ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/ref/array/with/key", "/ref/array/without/key"]. + +schema("/object") -> + to_schema([ + {per_page, mk(range(1, 100), #{nullable => false, desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, nullable => false})}, + {inner_ref, mk(hoconsc:ref(?MODULE, good_ref), #{})} + ]); +schema("/nest/object") -> + to_schema([ + {per_page, mk(range(1, 100), #{desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, nullable => false})}, + {nest_object, [ + {good_nest_1, mk(integer(), #{})}, + {good_nest_2, mk(hoconsc:ref(?MODULE, good_ref), #{})} + ]}, + {inner_ref, mk(hoconsc:ref(?MODULE, good_ref), #{})} + ]); +schema("/ref/local") -> + to_schema(mk(hoconsc:ref(good_ref), #{})); +schema("/ref/remote") -> + to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "ref2"), #{})); +schema("/ref/bad") -> + to_schema(mk(hoconsc:ref(?MODULE, bad_ref), #{})); +schema("/ref/nest/ref") -> + to_schema(mk(hoconsc:ref(?MODULE, nest_ref), #{})); +schema("/ref/array/with/key") -> + to_schema([ + {per_page, mk(range(1, 100), #{desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, required => true})}, + {array_refs, mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{})} + ]); +schema("/ref/array/without/key") -> + to_schema(mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{})). + +to_schema(Body) -> + #{ + operationId => test, + post => #{requestBody => Body, responses => #{200 => <<"ok">>}} + }. + +fields(good_ref) -> + [ + {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, + {tag, mk(binary(), #{desc => <<"tag">>})} + ]; +fields(nest_ref) -> + [ + {env, mk(hoconsc:enum([test, dev, prod]), #{})}, + {another_ref, mk(hoconsc:ref(good_ref), #{desc => "nest ref"})} + ]; + +fields(bad_ref) -> %% don't support maps + #{ + username => mk(string(), #{}), + is_admin => mk(boolean(), #{}) + }. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl new file mode 100644 index 000000000..80796cc2a --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -0,0 +1,292 @@ +-module(emqx_swagger_response_SUITE). + +-behaviour(minirest_api). +-behaviour(hocon_schema). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-import(hoconsc, [mk/2]). + +-export([all/0, suite/0, groups/0]). +-export([paths/0, api_spec/0, schema/1, fields/1]). +-export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, + t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, + t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, + t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). + +all() -> [{group, spec}]. +suite() -> [{timetrap, {minutes, 1}}]. +groups() -> [ + {spec, [parallel], [ + t_api_spec, t_simple_binary, t_object, t_nest_object, + t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, + t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, + t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} +]. + +t_simple_binary(_config) -> + Path = "/simple/bin", + ExpectSpec = #{description => <<"binary ok">>}, + ExpectRefs = [], + validate(Path, ExpectSpec, ExpectRefs), + ok. + +t_object(_config) -> + Path = "/object", + Object = + #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{required => [<<"timeout">>, <<"per_page">>], + <<"properties">> => [ + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}], + <<"type">> => object}}}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_nest_object(_Config) -> + Path = "/nest/object", + Object = + #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => + #{required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [ + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"nest_object">>, #{<<"type">> => object, <<"properties">> => [ + {<<"good_nest_1">>, #{example => 100, type => integer}}, + {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>} + }]}}, + {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}] + }}}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_empty(_Config) -> + ?assertThrow({error, + #{msg := <<"Object only supports not empty proplists">>, + args := [], module := ?MODULE}}, validate("/empty", error, [])), + ok. + +t_raw_local_ref(_Config) -> + Path = "/raw/ref/local", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_raw_remote_ref(_Config) -> + Path = "/raw/ref/remote", + Object = #{<<"content">> => + #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/remote.ref1">>}}}}, + ExpectRefs = [{emqx_swagger_remote_schema, "ref1"}], + validate(Path, Object, ExpectRefs), + ok. + +t_local_ref(_Config) -> + Path = "/ref/local", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_remote_ref(_Config) -> + Path = "/ref/remote", + Object = #{<<"content">> => + #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/remote.ref1">>}}}}, + ExpectRefs = [{emqx_swagger_remote_schema, "ref1"}], + validate(Path, Object, ExpectRefs), + ok. + +t_bad_ref(_Config) -> + Path = "/ref/bad", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => + #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.bad_ref">>}}}}, + ExpectRefs = [{?MODULE, bad_ref}], + ?assertThrow({error, #{module := ?MODULE, msg := <<"Object only supports not empty proplists">>}}, + validate(Path, Object, ExpectRefs)), + ok. + +t_none_ref(_Config) -> + Path = "/ref/none", + ?assertThrow({error, #{mfa := {?MODULE, schema, ["/ref/none"]}, + reason := function_clause}}, validate(Path, #{}, [])), + ok. + +t_nest_ref(_Config) -> + Path = "/ref/nest/ref", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.nest_ref">>}}}}, + ExpectRefs = [{?MODULE, nest_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_ref_array_with_key(_Config) -> + Path = "/ref/array/with/key", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [ + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"assert">>, #{description => <<"money">>, example => 3.14159,type => number}}, + {<<"number_ex">>, #{description => <<"number example">>, example => 42,type => number}}, + {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>,type => number}}, + {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>,type => string}}, + {<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}}, + {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}} + ]} + }}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. + +t_ref_array_without_key(_Config) -> + Path = "/ref/array/without/key", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, + type => array}}}}, + ExpectRefs = [{?MODULE, good_ref}], + validate(Path, Object, ExpectRefs), + ok. +t_hocon_schema_function(_Config) -> + Path = "/ref/hocon/schema/function", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => + #{<<"$ref">> => <<"#/components/schemas/remote.root">>}}}}, + ExpectComponents = [ + #{<<"remote.ref1">> => #{<<"type">> => object, + <<"properties">> => [ + {<<"protocol">>, #{enum => [http, https], type => string}}, + {<<"port">>, #{default => 18083, example => 100, type => integer}}] + }}, + #{<<"remote.ref2">> => #{<<"type">> => object, + <<"properties">> => [ + {<<"page">>, #{description => <<"good page">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/remote.ref3">>}} + ] + }}, + #{<<"remote.ref3">> => #{<<"type">> => object, + <<"properties">> => [ + {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"version">>, #{description => "a good version", example => <<"1.0.0">>, type => string}}] + }}, + #{<<"remote.root">> => #{required => [<<"default_password">>, <<"default_username">>], + <<"properties">> => [{<<"listeners">>, #{items => + #{<<"oneOf">> => + [#{<<"$ref">> => <<"#/components/schemas/remote.ref2">>}, + #{<<"$ref">> => <<"#/components/schemas/remote.ref1">>}]}, type => array}}, + {<<"default_username">>, + #{default => <<"admin">>, example => <<"string example">>, type => string}}, + {<<"default_password">>, #{default => <<"public">>, example => <<"string example">>, type => string}}, + {<<"sample_interval">>, #{default => <<"10s">>, example => <<"1h">>, type => string}}, + {<<"token_expired_time">>, #{default => <<"30m">>, example => <<"12m">>, type => string}}], + <<"type">> => object}}], + ExpectRefs = [{emqx_swagger_remote_schema, "root"}], + {_, Components} = validate(Path, Object, ExpectRefs), + ?assertEqual(ExpectComponents, Components), + ok. + +t_api_spec(_Config) -> + emqx_dashboard_swagger:spec(?MODULE), + ok. + +api_spec() -> emqx_dashboard_swagger:spec(?MODULE). + +paths() -> + ["/simple/bin", "/object", "/nest/object", "/ref/local", + "/ref/nest/ref", "/raw/ref/local", "/raw/ref/remote", + "/ref/array/with/key", "/ref/array/without/key", + "/ref/hocon/schema/function"]. + +schema("/simple/bin") -> + to_schema(<<"binary ok">>); +schema("/object") -> + Object = [ + {per_page, mk(range(1, 100), #{nullable => false, desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, nullable => false})}, + {inner_ref, mk(hoconsc:ref(?MODULE, good_ref), #{})} + ], + to_schema(Object); +schema("/nest/object") -> + Response = [ + {per_page, mk(range(1, 100), #{desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, nullable => false})}, + {nest_object, [ + {good_nest_1, mk(integer(), #{})}, + {good_nest_2, mk(hoconsc:ref(?MODULE, good_ref), #{})} + ]}, + {inner_ref, mk(hoconsc:ref(?MODULE, good_ref), #{})}], + to_schema(Response); +schema("/empty") -> + to_schema([]); +schema("/raw/ref/local") -> + to_schema(hoconsc:ref(good_ref)); +schema("/raw/ref/remote") -> + to_schema(hoconsc:ref(emqx_swagger_remote_schema, "ref1")); +schema("/ref/local") -> + to_schema(mk(hoconsc:ref(good_ref), #{})); +schema("/ref/remote") -> + to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "ref1"), #{})); +schema("/ref/bad") -> + to_schema(mk(hoconsc:ref(?MODULE, bad_ref), #{})); +schema("/ref/nest/ref") -> + to_schema(mk(hoconsc:ref(?MODULE, nest_ref), #{})); +schema("/ref/array/with/key") -> + to_schema([ + {per_page, mk(range(1, 100), #{desc => <<"good per page desc">>})}, + {timeout, mk(hoconsc:union([infinity, emqx_schema:duration_s()]), + #{default => 5, required => true})}, + {assert, mk(float(), #{desc => <<"money">>})}, + {number_ex, mk(number(), #{desc => <<"number example">>})}, + {percent_ex, mk(emqx_schema:percent(), #{desc => <<"percent example">>})}, + {duration_ms_ex, mk(emqx_schema:duration_ms(), #{desc => <<"duration ms example">>})}, + {atom_ex, mk(atom(), #{desc => <<"atom ex">>})}, + {array_refs, mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{})} + ]); +schema("/ref/array/without/key") -> + to_schema(mk(hoconsc:array(hoconsc:ref(?MODULE, good_ref)), #{})); +schema("/ref/hocon/schema/function") -> + to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "root"), #{})). + +validate(Path, ExpectObject, ExpectRefs) -> + {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), + ?assertEqual(test, OperationId), + Response = maps:get(responses, maps:get(post, Spec)), + ?assertEqual(ExpectObject, maps:get(<<"200">>, Response)), + ?assertEqual(ExpectObject, maps:get(<<"201">>, Response)), + ?assertEqual(#{}, maps:without([<<"201">>, <<"200">>], Response)), + ?assertEqual(ExpectRefs, Refs), + {Spec, emqx_dashboard_swagger:components(Refs)}. + +to_schema(Object) -> + #{ + operationId => test, + post => #{responses => #{200 => Object, 201 => Object}} + }. + +fields(good_ref) -> + [ + {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, + {tag, mk(binary(), #{desc => <<"tag">>})} + ]; +fields(nest_ref) -> + [ + {env, mk(hoconsc:enum([test, dev, prod]), #{})}, + {another_ref, mk(hoconsc:ref(good_ref), #{desc => "nest ref"})} + ]; + +fields(bad_ref) -> %% don't support maps + #{ + username => mk(string(), #{}), + is_admin => mk(boolean(), #{}) + }. diff --git a/rebar.config b/rebar.config index 310cd82ec..54e6d23f8 100644 --- a/rebar.config +++ b/rebar.config @@ -52,7 +52,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.3"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.4"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, "0.3.3"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} @@ -61,7 +61,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.19.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From 39423665e93456d10ba433aeaa30417b7ddd2284 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 22 Sep 2021 09:07:14 +0800 Subject: [PATCH 71/84] chore(swagger): check hocon schema with #{override_env => false} --- .../src/emqx_dashboard_swagger.erl | 6 +++--- .../test/emqx_swagger_remote_schema.erl | 3 +-- .../test/emqx_swagger_requestBody_SUITE.erl | 8 ++++---- .../test/emqx_swagger_response_SUITE.erl | 20 +++++++++---------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 517fe5bb5..99525a89a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -85,11 +85,11 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, BindingsAcc, QueryStr Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, case hocon_schema:field_schema(Type, in) of path -> - NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true}), + NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true, override_env => false}), NewBindingsAcc = maps:merge(BindingsAcc, NewBindings), check_parameter(Spec, Bindings, QueryStr, NewBindingsAcc, QueryStrAcc); query -> - NewQueryStr = hocon_schema:check_plain(Schema, QueryStr), + NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}), NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), check_parameter(Spec, Bindings, QueryStr, BindingsAcc, NewQueryStrAcc) end. @@ -102,7 +102,7 @@ check_requestBody(#{body := Body}, Schema, Module, true) -> _ -> Type0 end, NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]}, - #{<<"root">> := NewBody} = hocon_schema:check_plain(NewSchema, #{<<"root">> => Body}), + #{<<"root">> := NewBody} = hocon_schema:check_plain(NewSchema, #{<<"root">> => Body}, #{override_env => false}), NewBody; %% TODO not support nest object check yet, please use ref! %% RequestBody = [ {per_page, mk(integer(), #{}}, diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl index a8602a79e..91ef5a557 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl @@ -17,9 +17,8 @@ -include_lib("typerefl/include/types.hrl"). --export([ roots/0, fields/1,namespace/0]). +-export([ roots/0, fields/1]). -import(hoconsc, [mk/2]). -namespace() -> <<"remote">>. roots() -> ["root"]. fields("root") -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 462b20fd4..49dca926f 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -91,15 +91,15 @@ t_remote_ref(_Config) -> Spec = #{ post => #{parameters => [], requestBody => #{<<"content">> => #{<<"application/json">> => - #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/remote.ref2">>}}}}, + #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{emqx_swagger_remote_schema, "ref2"}], {_, Components} = validate("/ref/remote", Spec, Refs), ExpectComponents = [ - #{<<"remote.ref2">> => #{<<"properties">> => [ + #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"properties">> => [ {<<"page">>, #{description => <<"good page">>,example => 1, maximum => 100,minimum => 1,type => integer}}, - {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/remote.ref3">>}}], <<"type">> => object}}, - #{<<"remote.ref3">> => #{<<"properties">> => [ + {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}}, + #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"properties">> => [ {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, {<<"version">>, #{description => "a good version", example => <<"1.0.0">>,type => string}}], <<"type">> => object}}], diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 80796cc2a..c2140d2c0 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -84,7 +84,7 @@ t_raw_remote_ref(_Config) -> Path = "/raw/ref/remote", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ - <<"$ref">> => <<"#/components/schemas/remote.ref1">>}}}}, + <<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}}}}, ExpectRefs = [{emqx_swagger_remote_schema, "ref1"}], validate(Path, Object, ExpectRefs), ok. @@ -101,7 +101,7 @@ t_remote_ref(_Config) -> Path = "/ref/remote", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ - <<"$ref">> => <<"#/components/schemas/remote.ref1">>}}}}, + <<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}}}}, ExpectRefs = [{emqx_swagger_remote_schema, "ref1"}], validate(Path, Object, ExpectRefs), ok. @@ -159,29 +159,29 @@ t_ref_array_without_key(_Config) -> t_hocon_schema_function(_Config) -> Path = "/ref/hocon/schema/function", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => - #{<<"$ref">> => <<"#/components/schemas/remote.root">>}}}}, + #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.root">>}}}}, ExpectComponents = [ - #{<<"remote.ref1">> => #{<<"type">> => object, + #{<<"emqx_swagger_remote_schema.ref1">> => #{<<"type">> => object, <<"properties">> => [ {<<"protocol">>, #{enum => [http, https], type => string}}, {<<"port">>, #{default => 18083, example => 100, type => integer}}] }}, - #{<<"remote.ref2">> => #{<<"type">> => object, + #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"type">> => object, <<"properties">> => [ {<<"page">>, #{description => <<"good page">>, example => 1, maximum => 100, minimum => 1, type => integer}}, - {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/remote.ref3">>}} + {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}} ] }}, - #{<<"remote.ref3">> => #{<<"type">> => object, + #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object, <<"properties">> => [ {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, {<<"version">>, #{description => "a good version", example => <<"1.0.0">>, type => string}}] }}, - #{<<"remote.root">> => #{required => [<<"default_password">>, <<"default_username">>], + #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>], <<"properties">> => [{<<"listeners">>, #{items => #{<<"oneOf">> => - [#{<<"$ref">> => <<"#/components/schemas/remote.ref2">>}, - #{<<"$ref">> => <<"#/components/schemas/remote.ref1">>}]}, type => array}}, + [#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}, + #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, type => array}}, {<<"default_username">>, #{default => <<"admin">>, example => <<"string example">>, type => string}}, {<<"default_password">>, #{default => <<"public">>, example => <<"string example">>, type => string}}, From ab2cdfeab1c9b15aab47ecb8928e0092027fe38e Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Wed, 22 Sep 2021 10:04:00 +0800 Subject: [PATCH 72/84] fix: delayed message format & message id parameter name (#5762) --- apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_modules/src/emqx_delayed_api.erl | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 0b1f00e14..823424c1f 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -143,7 +143,7 @@ format_delayed(#delayed_message{key = {ExpectTimeStamp, Id}, delayed = Delayed, ExpectTime = to_rfc3339(ExpectTimeStamp), RemainingTime = ExpectTimeStamp - erlang:system_time(second), Result = #{ - id => emqx_guid:to_hexstr(Id), + msgid => emqx_guid:to_hexstr(Id), publish_at => PublishTime, delayed_interval => Delayed, delayed_remaining => RemainingTime, diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index d78b7ddcb..96589bf06 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -59,7 +59,7 @@ properties() -> PayloadDesc = io_lib:format("Payload, base64 encode. Payload will be ~p if length large than ~p", [?PAYLOAD_TOO_LARGE, ?MAX_PAYLOAD_LENGTH]), properties([ - {id, integer, <<"Message Id (MQTT message id hash)">>}, + {msgid, integer, <<"Message Id">>}, {publish_at, string, <<"Client publish message time, rfc 3339">>}, {delayed_interval, integer, <<"Delayed interval, second">>}, {delayed_remaining, integer, <<"Delayed remaining, second">>}, @@ -73,7 +73,7 @@ properties() -> parameters() -> [#{ - name => id, + name => msgid, in => path, schema => #{type => string}, required => true @@ -129,7 +129,7 @@ delayed_message_api() -> } } }, - {"/mqtt/delayed/messages/:id", Metadata, delayed_message}. + {"/mqtt/delayed/messages/:msgid", Metadata, delayed_message}. %%-------------------------------------------------------------------- %% HTTP API @@ -143,7 +143,7 @@ status(put, #{body := Body}) -> delayed_messages(get, #{query_string := Qs}) -> {200, emqx_delayed:list(Qs)}. -delayed_message(get, #{bindings := #{id := Id}}) -> +delayed_message(get, #{bindings := #{msgid := Id}}) -> case emqx_delayed:get_delayed_message(Id) of {ok, Message} -> Payload = maps:get(payload, Message), @@ -157,7 +157,7 @@ delayed_message(get, #{bindings := #{id := Id}}) -> Message = iolist_to_binary(io_lib:format("Message ID ~p not found", [Id])), {404, #{code => ?MESSAGE_ID_NOT_FOUND, message => Message}} end; -delayed_message(delete, #{bindings := #{id := Id}}) -> +delayed_message(delete, #{bindings := #{msgid := Id}}) -> _ = emqx_delayed:delete_delayed_message(Id), {200}. From 00d7d66871c0c8b5011f25a3de3d26f3d004ade0 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 22 Sep 2021 11:28:31 +0800 Subject: [PATCH 73/84] chore(authz): rename mongo to mongodb --- apps/emqx_authz/etc/emqx_authz.conf | 2 +- apps/emqx_authz/src/emqx_authz.erl | 6 +++-- apps/emqx_authz/src/emqx_authz_api_schema.erl | 12 +++++----- ...authz_mongo.erl => emqx_authz_mongodb.erl} | 4 ++-- apps/emqx_authz/src/emqx_authz_schema.erl | 22 ++++++++++++------- apps/emqx_authz/test/emqx_authz_SUITE.erl | 20 ++++++++--------- .../test/emqx_authz_api_sources_SUITE.erl | 22 +++++++++---------- ...SUITE.erl => emqx_authz_mongodb_SUITE.erl} | 4 ++-- 8 files changed, 50 insertions(+), 42 deletions(-) rename apps/emqx_authz/src/{emqx_authz_mongo.erl => emqx_authz_mongodb.erl} (98%) rename apps/emqx_authz/test/{emqx_authz_mongo_SUITE.erl => emqx_authz_mongodb_SUITE.erl} (98%) diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 19bb2737b..1111a7819 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -46,7 +46,7 @@ authorization { # cmd: "HGETALL mqtt_authz:%u" # }, # { - # type: mongo + # type: mongodb # mongo_type: single # server: "127.0.0.1:27017" # pool_size: 1 diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index cf7447f6a..e7ccbe5b0 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -39,7 +39,7 @@ -export([post_config_update/4, pre_config_update/2]). -define(CONF_KEY_PATH, [authorization, sources]). --define(SOURCE_TYPES, [file, http, mongo, mysql, postgresql, redis]). +-define(SOURCE_TYPES, [file, http, mongodb, mysql, postgresql, redis]). -spec(register_metrics() -> ok). register_metrics() -> @@ -300,7 +300,7 @@ init_source(#{enable := true, init_source(#{enable := true, type := DB } = Source) when DB =:= redis; - DB =:= mongo -> + DB =:= mongodb -> case create_resource(Source) of {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id}} @@ -407,6 +407,8 @@ create_resource(#{type := DB} = Source) -> authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). +connector_module(mongodb) -> + emqx_connector_mongo; connector_module(postgresql) -> emqx_connector_pgsql; connector_module(Type) -> diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 4fd2f8ca1..d99c5acb5 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -132,8 +132,8 @@ definitions() -> properties => #{ type => #{ type => string, - enum => [<<"mongo">>], - example => <<"mongo">> + enum => [<<"mongodb">>], + example => <<"mongodb">> }, enable => #{ type => boolean, @@ -188,8 +188,8 @@ definitions() -> properties => #{ type => #{ type => string, - enum => [<<"mongo">>], - example => <<"mongo">> + enum => [<<"mongodb">>], + example => <<"mongodb">> }, enable => #{ type => boolean, @@ -245,8 +245,8 @@ definitions() -> properties => #{ type => #{ type => string, - enum => [<<"mongo">>], - example => <<"mongo">> + enum => [<<"mongodb">>], + example => <<"mongodb">> }, enable => #{ type => boolean, diff --git a/apps/emqx_authz/src/emqx_authz_mongo.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl similarity index 98% rename from apps/emqx_authz/src/emqx_authz_mongo.erl rename to apps/emqx_authz/src/emqx_authz_mongodb.erl index 68808c20b..6c0fb126a 100644 --- a/apps/emqx_authz/src/emqx_authz_mongo.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authz_mongo). +-module(emqx_authz_mongodb). -include("emqx_authz.hrl"). -include_lib("emqx/include/emqx.hrl"). @@ -31,7 +31,7 @@ -endif. description() -> - "AuthZ with Mongo". + "AuthZ with MongoDB". authorize(Client, PubSub, Topic, #{collection := Collection, diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index bda8d8e74..c880b7669 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -114,28 +114,34 @@ fields(http_post) -> } ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); fields(mongo_single) -> - connector_fields(mongo, single) ++ [ {collection, #{type => atom()}} , {selector, #{type => map()}} - ]; + , {type, #{type => mongodb}} + , {enable, #{type => boolean(), + default => true}} + ] ++ emqx_connector_mongo:fields(single); fields(mongo_rs) -> - connector_fields(mongo, rs) ++ [ {collection, #{type => atom()}} , {selector, #{type => map()}} - ]; + , {type, #{type => mongodb}} + , {enable, #{type => boolean(), + default => true}} + ] ++ emqx_connector_mongo:fields(rs); fields(mongo_sharded) -> - connector_fields(mongo, sharded) ++ [ {collection, #{type => atom()}} , {selector, #{type => map()}} - ]; + , {type, #{type => mongodb}} + , {enable, #{type => boolean(), + default => true}} + ] ++ emqx_connector_mongo:fields(sharded); fields(mysql) -> connector_fields(mysql) ++ [ {query, query()} ]; fields(postgresql) -> - [ {type, #{type => postgresql}} + [ {query, query()} + , {type, #{type => postgresql}} , {enable, #{type => boolean(), default => true}} - , {query, query()} ] ++ emqx_connector_pgsql:fields(config); fields(redis_single) -> connector_fields(redis, single) ++ diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index bed20e0e4..bfdc131a0 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -67,7 +67,7 @@ init_per_testcase(_, Config) -> <<"method">> => <<"get">>, <<"request_timeout">> => 5000 }). --define(SOURCE2, #{<<"type">> => <<"mongo">>, +-define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, <<"mongo_type">> => <<"single">>, <<"server">> => <<"127.0.0.1:27017">>, @@ -128,7 +128,7 @@ t_update_source(_) -> {ok, _} = emqx_authz:update(tail, [?SOURCE6]), ?assertMatch([ #{type := http, enable := true} - , #{type := mongo, enable := true} + , #{type := mongodb, enable := true} , #{type := mysql, enable := true} , #{type := postgresql, enable := true} , #{type := redis, enable := true} @@ -136,14 +136,14 @@ t_update_source(_) -> ], emqx:get_config([authorization, sources], [])), {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, mongo}, ?SOURCE2#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, mongodb}, ?SOURCE2#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, postgresql}, ?SOURCE4#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}), ?assertMatch([ #{type := http, enable := false} - , #{type := mongo, enable := false} + , #{type := mongodb, enable := false} , #{type := mysql, enable := false} , #{type := postgresql, enable := false} , #{type := redis, enable := false} @@ -155,7 +155,7 @@ t_update_source(_) -> t_move_source(_) -> {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), ?assertMatch([ #{type := http} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := postgresql} , #{type := redis} @@ -165,7 +165,7 @@ t_move_source(_) -> {ok, _} = emqx_authz:move(postgresql, <<"top">>), ?assertMatch([ #{type := postgresql} , #{type := http} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := redis} , #{type := file} @@ -173,7 +173,7 @@ t_move_source(_) -> {ok, _} = emqx_authz:move(http, <<"bottom">>), ?assertMatch([ #{type := postgresql} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := redis} , #{type := file} @@ -183,19 +183,19 @@ t_move_source(_) -> {ok, _} = emqx_authz:move(mysql, #{<<"before">> => postgresql}), ?assertMatch([ #{type := mysql} , #{type := postgresql} - , #{type := mongo} + , #{type := mongodb} , #{type := redis} , #{type := file} , #{type := http} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(mongo, #{<<"after">> => http}), + {ok, _} = emqx_authz:move(mongodb, #{<<"after">> => http}), ?assertMatch([ #{type := mysql} , #{type := postgresql} , #{type := redis} , #{type := file} , #{type := http} - , #{type := mongo} + , #{type := mongodb} ], emqx_authz:lookup()), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index f2118c05f..a3c6e6e50 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -44,7 +44,7 @@ <<"method">> => <<"get">>, <<"request_timeout">> => 5000 }). --define(SOURCE2, #{<<"type">> => <<"mongo">>, +-define(SOURCE2, #{<<"type">> => <<"mongodb">>, <<"enable">> => true, <<"mongo_type">> => <<"sharded">>, <<"servers">> => [<<"127.0.0.1:27017">>, @@ -181,7 +181,7 @@ t_api(_) -> {ok, 200, Result2} = request(get, uri(["authorization", "sources"]), []), Sources = get_sources(Result2), ?assertMatch([ #{<<"type">> := <<"http">>} - , #{<<"type">> := <<"mongo">>} + , #{<<"type">> := <<"mongodb">>} , #{<<"type">> := <<"mysql">>} , #{<<"type">> := <<"postgresql">>} , #{<<"type">> := <<"redis">>} @@ -193,7 +193,7 @@ t_api(_) -> {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch(#{<<"type">> := <<"http">>, <<"enable">> := false}, jsx:decode(Result3)), - {ok, 204, _} = request(put, uri(["authorization", "sources", "mongo"]), + {ok, 204, _} = request(put, uri(["authorization", "sources", "mongodb"]), ?SOURCE2#{<<"ssl">> := #{ <<"enable">> => true, <<"cacertfile">> => <<"fake cacert file">>, @@ -201,8 +201,8 @@ t_api(_) -> <<"keyfile">> => <<"fake key file">>, <<"verify">> => false }}), - {ok, 200, Result4} = request(get, uri(["authorization", "sources", "mongo"]), []), - ?assertMatch(#{<<"type">> := <<"mongo">>, + {ok, 200, Result4} = request(get, uri(["authorization", "sources", "mongodb"]), []), + ?assertMatch(#{<<"type">> := <<"mongodb">>, <<"ssl">> := #{<<"enable">> := true, <<"cacertfile">> := <<"fake cacert file">>, <<"certfile">> := <<"fake cert file">>, @@ -225,7 +225,7 @@ t_api(_) -> t_move_source(_) -> {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5]), ?assertMatch([ #{type := http} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := postgresql} , #{type := redis} @@ -235,7 +235,7 @@ t_move_source(_) -> #{<<"position">> => <<"top">>}), ?assertMatch([ #{type := postgresql} , #{type := http} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := redis} ], emqx_authz:lookup()), @@ -243,7 +243,7 @@ t_move_source(_) -> {ok, 204, _} = request(post, uri(["authorization", "sources", "http", "move"]), #{<<"position">> => <<"bottom">>}), ?assertMatch([ #{type := postgresql} - , #{type := mongo} + , #{type := mongodb} , #{type := mysql} , #{type := redis} , #{type := http} @@ -253,18 +253,18 @@ t_move_source(_) -> #{<<"position">> => #{<<"before">> => <<"postgresql">>}}), ?assertMatch([ #{type := mysql} , #{type := postgresql} - , #{type := mongo} + , #{type := mongodb} , #{type := redis} , #{type := http} ], emqx_authz:lookup()), - {ok, 204, _} = request(post, uri(["authorization", "sources", "mongo", "move"]), + {ok, 204, _} = request(post, uri(["authorization", "sources", "mongodb", "move"]), #{<<"position">> => #{<<"after">> => <<"http">>}}), ?assertMatch([ #{type := mysql} , #{type := postgresql} , #{type := redis} , #{type := http} - , #{type := mongo} + , #{type := mongodb} ], emqx_authz:lookup()), ok. diff --git a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl similarity index 98% rename from apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl rename to apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index ec4c4f384..6925194bd 100644 --- a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -13,7 +13,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authz_mongo_SUITE). +-module(emqx_authz_mongodb_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -46,7 +46,7 @@ init_per_suite(Config) -> ok = emqx_ct_helpers:start_apps([emqx_authz]), {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), - Rules = [#{<<"type">> => <<"mongo">>, + Rules = [#{<<"type">> => <<"mongodb">>, <<"mongo_type">> => <<"single">>, <<"server">> => <<"127.0.0.1:27017">>, <<"pool_size">> => 1, From 96fad3d225440a91848af50d3b8e060ab2f556e9 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 22 Sep 2021 09:55:12 +0800 Subject: [PATCH 74/84] chore(swagger): support check_schema option, default is false. --- .../emqx_dashboard/src/emqx_dashboard_api.erl | 2 +- .../src/emqx_dashboard_swagger.erl | 26 ++++++++++++++++--- .../test/emqx_swagger_parameter_SUITE.erl | 8 +++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index deb7dc74c..874d2bf2b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -41,7 +41,7 @@ namespace() -> "dashboard". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> ["/login", "/logout", "/users", "/users/:username", "/users/:username/change_pwd"]. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 99525a89a..9033a9b40 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -4,7 +4,7 @@ -include_lib("hocon/include/hoconsc.hrl"). %% API --export([spec/1]). +-export([spec/1, spec/2]). -export([translate_req/2]). -ifdef(TEST). @@ -24,23 +24,37 @@ -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). -define(TO_COMPONENTS(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])). --spec(spec(module()) -> {list({Path, Specs, OperationId, Options}), list(Component)} when +%% @equiv spec(Module, #{check_schema => false}) +-spec(spec(module()) -> + {list({Path, Specs, OperationId, Options}), list(Component)} when Path :: string()|binary(), Specs :: map(), OperationId :: atom(), Options :: #{filter => fun((map(), #{module => module(), path => string(), method => atom()}) -> map())}, Component :: map()). -spec(Module) -> +spec(Module) -> spec(Module, #{check_schema => false}). + +-spec(spec(module(), #{check_schema => boolean()}) -> + {list({Path, Specs, OperationId, Options}), list(Component)} when + Path :: string()|binary(), + Specs :: map(), + OperationId :: atom(), + Options :: #{filter => fun((map(), + #{module => module(), path => string(), method => atom()}) -> map())}, + Component :: map()). +spec(Module, Options) -> Paths = apply(Module, paths, []), {ApiSpec, AllRefs} = lists:foldl(fun(Path, {AllAcc, AllRefsAcc}) -> {OperationId, Specs, Refs} = parse_spec_ref(Module, Path), - {[{Path, Specs, OperationId, ?DEFAULT_FILTER} | AllAcc], + CheckSchema = support_check_schema(Options), + {[{Path, Specs, OperationId, CheckSchema} | AllAcc], Refs ++ AllRefsAcc} end, {[], []}, Paths), {ApiSpec, components(lists:usort(AllRefs))}. + -spec(translate_req(#{binding => list(), query_string => list(), body => map()}, #{module => module(), path => string(), method => atom()}) -> {ok, #{binding => list(), query_string => list(), body => map()}}| @@ -59,6 +73,10 @@ translate_req(Request, #{module := Module, path := Path, method := Method}) -> {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~s : ~p", [Key, Reason]))} end. +support_check_schema(#{check_schema := true}) -> ?DEFAULT_FILTER; +support_check_schema(#{check_schema := Func})when is_function(Func, 2) -> #{filter => Func}; +support_check_schema(_) -> #{filter => undefined}. + parse_spec_ref(Module, Path) -> Schema = try diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 74cb53a26..a5c458ffa 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -162,7 +162,13 @@ t_in_mix_trans_error(_Config) -> ok. t_api_spec(_Config) -> - emqx_dashboard_swagger:spec(?MODULE), + {Spec, _Components} = emqx_dashboard_swagger:spec(?MODULE), + Filter = fun(V, S) -> lists:all(fun({_, _, _, #{filter := Filter}}) -> Filter =:= V end, S) end, + ?assertEqual(true, Filter(undefined, Spec)), + {Spec1, _Components1} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}), + ?assertEqual(true, Filter(fun emqx_dashboard_swagger:translate_req/2, Spec1)), + {Spec2, _Components2} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun emqx_dashboard_swagger:translate_req/2}), + ?assertEqual(true, Filter(fun emqx_dashboard_swagger:translate_req/2, Spec2)), ok. validate(Path, ExpectParams) -> From 502f962b4cea70f2990f245843c0f161f7b9970f Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 17 Sep 2021 13:46:25 +0800 Subject: [PATCH 75/84] refactor(bridge): change confs for channel from array to map --- apps/emqx_bridge/etc/emqx_bridge.conf | 18 ++-- apps/emqx_bridge/src/emqx_bridge_schema.erl | 4 +- .../src/emqx_connector_mqtt.erl | 82 +++++++++---------- .../src/mqtt/emqx_connector_mqtt_schema.erl | 7 +- 4 files changed, 52 insertions(+), 59 deletions(-) diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index fc050f4b8..e8af40341 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -25,25 +25,23 @@ # certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" # cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" # } -# ## we will create one MQTT connection for each element of the `ingress_channels` -# ingress_channels: [{ -# ## the `id` will be used as part of the clientid -# id = "pull_msgs_from_aws" +# ## We will create one MQTT connection for each element of the `ingress_channels` +# ## Syntax: ingress_channels. +# ingress_channels.pull_msgs_from_aws { # subscribe_remote_topic = "aws/#" # subscribe_qos = 1 # local_topic = "from_aws/${topic}" # payload = "${payload}" # qos = "${qos}" # retain = "${retain}" -# }] -# ## we will create one MQTT connection for each element of the `egress_channels` -# egress_channels: [{ -# ## the `id` will be used as part of the clientid -# id = "push_msgs_to_aws" +# } +# ## We will create one MQTT connection for each element of the `egress_channels` +# ## Syntax: egress_channels. +# egress_channels.push_msgs_to_aws { # subscribe_local_topic = "emqx/#" # remote_topic = "from_emqx/${topic}" # payload = "${payload}" # qos = 1 # retain = false -# }] +# } #} diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 94cdaa30b..87eb40372 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -5,9 +5,9 @@ %%====================================================================================== %% Hocon Schema Definitions -roots() -> ["bridges"]. +roots() -> [bridges]. -fields("bridges") -> +fields(bridges) -> [{mqtt, hoconsc:mk(hoconsc:map(name, hoconsc:ref(?MODULE, "mqtt_bridge")))}]; fields("mqtt_bridge") -> diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 125b5cbe4..36ffd5706 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -88,20 +88,20 @@ on_start(InstId, Conf) -> logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]), NamePrefix = binary_to_list(InstId), BasicConf = basic_config(Conf), - InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, sub_bridges => []}}, - InOutConfigs = check_channel_id_dup(maps:get(ingress_channels, Conf, []) - ++ maps:get(egress_channels, Conf, [])), + InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, channels => []}}, + InOutConfigs = taged_map_list(ingress_channels, maps:get(ingress_channels, Conf, #{})) + ++ taged_map_list(egress_channels, maps:get(egress_channels, Conf, #{})), lists:foldl(fun (_InOutConf, {error, Reason}) -> {error, Reason}; - (InOutConf, {ok, #{sub_bridges := SubBridges} = Res}) -> + (InOutConf, {ok, #{channels := SubBridges} = Res}) -> case create_channel(InOutConf, NamePrefix, BasicConf) of {error, Reason} -> {error, Reason}; - {ok, Name} -> {ok, Res#{sub_bridges => [Name | SubBridges]}} + {ok, Name} -> {ok, Res#{channels => [Name | SubBridges]}} end end, InitRes, InOutConfigs). -on_stop(InstId, #{sub_bridges := NameList}) -> +on_stop(InstId, #{channels := NameList}) -> logger:info("stopping mqtt connector: ~p", [InstId]), lists:foreach(fun(Name) -> remove_channel(Name) @@ -117,63 +117,53 @@ on_query(InstId, {publish_to_local, Msg}, _AfterQuery, _State) -> on_query(InstId, {publish_to_remote, Msg}, _AfterQuery, _State) -> logger:debug("publish to remote node, connector: ~p, msg: ~p", [InstId, Msg]). -on_health_check(_InstId, #{sub_bridges := NameList} = State) -> +on_health_check(_InstId, #{channels := NameList} = State) -> Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList], case lists:all(fun({_, pong}) -> true; ({_, _}) -> false end, Results) of true -> {ok, State}; - false -> {error, {some_sub_bridge_down, Results}, State} + false -> {error, {some_channel_down, Results}, State} end. -check_channel_id_dup(Confs) -> - lists:foreach(fun(#{id := Id}) -> - case length([Id || #{id := Id0} <- Confs, Id0 == Id]) of - 1 -> ok; - L when L > 1 -> error({mqtt_bridge_conf, {duplicate_id_found, Id}}) - end - end, Confs), - Confs. - -%% this is an `ingress_channels` bridge -create_channel(#{subscribe_remote_topic := RemoteT, local_topic := LocalT, id := Id} = InConf, - NamePrefix, BasicConf) -> - Name = bridge_name(NamePrefix, Id), +create_channel({{ingress_channels, Id}, #{subscribe_remote_topic := RemoteT, + local_topic := LocalT} = Conf}, NamePrefix, BasicConf) -> + Name = ingress_channel_name(NamePrefix, Id), logger:info("creating ingress channel ~p, remote ~s -> local ~s", [Name, RemoteT, LocalT]), - create_sub_bridge(BasicConf#{ + do_create_channel(BasicConf#{ name => Name, - clientid => clientid(Id), - subscriptions => InConf, forwards => undefined}); -%% this is an `egress_channels` bridge -create_channel(#{subscribe_local_topic := LocalT, remote_topic := RemoteT, id := Id} = OutConf, - NamePrefix, BasicConf) -> - Name = bridge_name(NamePrefix, Id), + clientid => clientid(Name), + subscriptions => Conf, forwards => undefined}); + +create_channel({{egress_channels, Id}, #{subscribe_local_topic := LocalT, + remote_topic := RemoteT} = Conf}, NamePrefix, BasicConf) -> + Name = egress_channel_name(NamePrefix, Id), logger:info("creating egress channel ~p, local ~s -> remote ~s", [Name, LocalT, RemoteT]), - create_sub_bridge(BasicConf#{ - name => bridge_name(NamePrefix, Id), - clientid => clientid(Id), - subscriptions => undefined, forwards => OutConf}). + do_create_channel(BasicConf#{ + name => Name, + clientid => clientid(Name), + subscriptions => undefined, forwards => Conf}). -remove_channel(BridgeName) -> - logger:info("removing channel ~p", [BridgeName]), - case ?MODULE:drop_bridge(BridgeName) of +remove_channel(ChannelName) -> + logger:info("removing channel ~p", [ChannelName]), + case ?MODULE:drop_bridge(ChannelName) of ok -> ok; {error, not_found} -> ok; {error, Reason} -> - logger:error("stop channel ~p failed, error: ~p", [BridgeName, Reason]) + logger:error("stop channel ~p failed, error: ~p", [ChannelName, Reason]) end. -create_sub_bridge(#{name := Name} = Conf) -> +do_create_channel(#{name := Name} = Conf) -> case ?MODULE:create_bridge(Conf) of {ok, _Pid} -> - start_sub_bridge(Name); + start_channel(Name); {error, {already_started, _Pid}} -> {ok, Name}; {error, Reason} -> {error, Reason} end. -start_sub_bridge(Name) -> +start_channel(Name) -> case emqx_connector_mqtt_worker:ensure_started(Name) of ok -> {ok, Name}; {error, Reason} -> {error, Reason} @@ -210,11 +200,19 @@ basic_config(#{ if_record_metrics => true }. -bridge_name(Prefix, Id) -> - list_to_atom(str(Prefix) ++ ":" ++ str(Id)). +taged_map_list(Tag, Map) -> + [{{Tag, K}, V} || {K, V} <- maps:to_list(Map)]. + +ingress_channel_name(Prefix, Id) -> + channel_name("ingress_channels", Prefix, Id). +egress_channel_name(Prefix, Id) -> + channel_name("egress_channels", Prefix, Id). + +channel_name(Type, Prefix, Id) -> + list_to_atom(str(Prefix) ++ ":" ++ Type ++ ":" ++ str(Id)). clientid(Id) -> - list_to_binary(str(Id) ++ ":" ++ emqx_misc:gen_id(16)). + list_to_binary(str(Id) ++ ":" ++ emqx_misc:gen_id(8)). str(A) when is_atom(A) -> atom_to_list(A); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 28995b666..89fe5f581 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -38,8 +38,8 @@ fields("config") -> , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} , {max_inflight, hoconsc:mk(integer(), #{default => 32})} , {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))} - , {ingress_channels, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "ingress_channels")), #{default => []})} - , {egress_channels, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "egress_channels")), #{default => []})} + , {ingress_channels, hoconsc:mk(hoconsc:map(id, hoconsc:ref(?MODULE, "ingress_channels")), #{default => []})} + , {egress_channels, hoconsc:mk(hoconsc:map(id, hoconsc:ref(?MODULE, "egress_channels")), #{default => []})} ] ++ emqx_connector_schema_lib:ssl_fields(); fields("ingress_channels") -> @@ -61,9 +61,6 @@ fields("replayq") -> ]. common_inout_confs() -> - [{id, #{type => binary(), nullable => false}}] ++ publish_confs(). - -publish_confs() -> [ {qos, hoconsc:mk(qos(), #{default => <<"${qos}">>})} , {retain, hoconsc:mk(hoconsc:union([boolean(), binary()]), #{default => <<"${retain}">>})} , {payload, hoconsc:mk(binary(), #{default => <<"${payload}">>})} From c0258e1be6db3c1a47b1299b4c8e57e0667a5b3a Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 17 Sep 2021 18:56:33 +0800 Subject: [PATCH 76/84] feat(bridge): let mqtt bridge work with rules --- .../src/emqx_connector_mqtt.erl | 40 +++++++++++------- .../src/mqtt/emqx_connector_mqtt_mod.erl | 10 ++++- .../src/mqtt/emqx_connector_mqtt_msg.erl | 14 +++---- .../src/mqtt/emqx_connector_mqtt_schema.erl | 8 ++-- .../src/mqtt/emqx_connector_mqtt_worker.erl | 41 ++++++++----------- 5 files changed, 63 insertions(+), 50 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 36ffd5706..a03c888d3 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -28,6 +28,8 @@ , bridges/0 ]). +-export([on_message_received/2]). + %% callbacks of behaviour emqx_resource -export([ on_start/2 , on_stop/2 @@ -83,6 +85,12 @@ drop_bridge(Name) -> {error, Error} end. +%% =================================================================== +%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called +%% if the bridge received msgs from the remote broker. +on_message_received(Msg, ChannelName) -> + emqx:run_hook(ChannelName, [Msg]). + %% =================================================================== on_start(InstId, Conf) -> logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]), @@ -112,10 +120,9 @@ on_stop(InstId, #{channels := NameList}) -> on_query(_InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, baisc_conf := BasicConf}) -> create_channel(Conf, Prefix, BasicConf); -on_query(InstId, {publish_to_local, Msg}, _AfterQuery, _State) -> - logger:debug("publish to local node, connector: ~p, msg: ~p", [InstId, Msg]); -on_query(InstId, {publish_to_remote, Msg}, _AfterQuery, _State) -> - logger:debug("publish to remote node, connector: ~p, msg: ~p", [InstId, Msg]). +on_query(_InstId, {send_to_remote, ChannelName, Msg}, _AfterQuery, _State) -> + logger:debug("send msg to remote node on channel: ~p, msg: ~p", [ChannelName, Msg]), + emqx_connector_mqtt_worker:send_to_remote(ChannelName, Msg). on_health_check(_InstId, #{channels := NameList} = State) -> Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList], @@ -124,25 +131,30 @@ on_health_check(_InstId, #{channels := NameList} = State) -> false -> {error, {some_channel_down, Results}, State} end. -create_channel({{ingress_channels, Id}, #{subscribe_remote_topic := RemoteT, - local_topic := LocalT} = Conf}, NamePrefix, BasicConf) -> +create_channel({{ingress_channels, Id}, #{subscribe_remote_topic := RemoteT} = Conf}, + NamePrefix, BasicConf) -> + LocalT = maps:get(local_topic, Conf, undefined), Name = ingress_channel_name(NamePrefix, Id), - logger:info("creating ingress channel ~p, remote ~s -> local ~s", - [Name, RemoteT, LocalT]), + logger:info("creating ingress channel ~p, remote ~s -> local ~s", [Name, RemoteT, LocalT]), do_create_channel(BasicConf#{ name => Name, clientid => clientid(Name), - subscriptions => Conf, forwards => undefined}); + subscriptions => Conf#{ + local_topic => LocalT, + on_message_received => {fun ?MODULE:on_message_received/2, [Name]} + }, + forwards => undefined}); -create_channel({{egress_channels, Id}, #{subscribe_local_topic := LocalT, - remote_topic := RemoteT} = Conf}, NamePrefix, BasicConf) -> +create_channel({{egress_channels, Id}, #{remote_topic := RemoteT} = Conf}, + NamePrefix, BasicConf) -> + LocalT = maps:get(subscribe_local_topic, Conf, undefined), Name = egress_channel_name(NamePrefix, Id), - logger:info("creating egress channel ~p, local ~s -> remote ~s", - [Name, LocalT, RemoteT]), + logger:info("creating egress channel ~p, local ~s -> remote ~s", [Name, LocalT, RemoteT]), do_create_channel(BasicConf#{ name => Name, clientid => clientid(Name), - subscriptions => undefined, forwards => Conf}). + subscriptions => undefined, + forwards => Conf#{subscribe_local_topic => LocalT}}). remove_channel(ChannelName) -> logger:info("removing channel ~p", [ChannelName]), diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 3de7feac4..8b0aa5051 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -159,9 +159,15 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> handle_publish(Msg, undefined) -> ?LOG(error, "cannot publish to local broker as 'bridge.mqtt..in' not configured, msg: ~p", [Msg]); -handle_publish(Msg, Vars) -> +handle_publish(Msg, #{on_message_received := {OnMsgRcvdFunc, Args}} = Vars) -> ?LOG(debug, "publish to local broker, msg: ~p, vars: ~p", [Msg, Vars]), - emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)). + emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1), + _ = erlang:apply(OnMsgRcvdFunc, [Msg, Args]), + case maps:get(local_topic, Vars, undefined) of + undefined -> ok; + _Topic -> + emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)) + end. handle_disconnected(Reason, Parent) -> Parent ! {disconnected, self(), Reason}. diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 5f076ed9e..6009cc084 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -36,17 +36,15 @@ -type variables() :: #{ mountpoint := undefined | binary(), - topic := binary(), + remote_topic := binary(), qos := original | integer(), retain := original | boolean(), payload := binary() }. make_pub_vars(_, undefined) -> undefined; -make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, remote_topic := Topic} = Conf) -> - Conf#{topic => Topic, mountpoint => Mountpoint}; -make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, local_topic := Topic} = Conf) -> - Conf#{topic => Topic, mountpoint => Mountpoint}. +make_pub_vars(Mountpoint, Conf) when is_map(Conf) -> + Conf#{mountpoint => Mountpoint}. %% @doc Make export format: %% 1. Mount topic to a prefix @@ -61,7 +59,7 @@ to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> Retain0 = maps:get(retain, Flags0, false), MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)), to_remote_msg(MapMsg, Vars); -to_remote_msg(MapMsg, #{topic := TopicToken, payload := PayloadToken, +to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken, qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> Topic = replace_vars_in_str(TopicToken, MapMsg), Payload = replace_vars_in_str(PayloadToken, MapMsg), @@ -77,7 +75,7 @@ to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> %% published from remote node over a MQTT connection to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, - #{topic := TopicToken, payload := PayloadToken, + #{local_topic := TopicToken, payload := PayloadToken, qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> Topic = replace_vars_in_str(TopicToken, MapMsg), Payload = replace_vars_in_str(PayloadToken, MapMsg), @@ -115,6 +113,8 @@ from_binary(Bin) -> binary_to_term(Bin). %% Count only the topic length + payload size -spec estimate_size(msg()) -> integer(). estimate_size(#message{topic = Topic, payload = Payload}) -> + size(Topic) + size(Payload); +estimate_size(#{topic := Topic, payload := Payload}) -> size(Topic) + size(Payload). set_headers(undefined, Msg) -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 89fe5f581..a00b76b97 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -43,13 +43,15 @@ fields("config") -> ] ++ emqx_connector_schema_lib:ssl_fields(); fields("ingress_channels") -> - [ {subscribe_remote_topic, #{type => binary(), nullable => false}} - , {local_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} + %% the message maybe subscribed by rules, in this case 'local_topic' is not necessary + [ {subscribe_remote_topic, hoconsc:mk(binary(), #{nullable => false})} + , {local_topic, hoconsc:mk(binary())} , {subscribe_qos, hoconsc:mk(qos(), #{default => 1})} ] ++ common_inout_confs(); fields("egress_channels") -> - [ {subscribe_local_topic, #{type => binary(), nullable => false}} + %% the message maybe sent from rules, in this case 'subscribe_local_topic' is not necessary + [ {subscribe_local_topic, hoconsc:mk(binary())} , {remote_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} ] ++ common_inout_confs(); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index e5a1a807f..c98efd322 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -87,6 +87,7 @@ , ensure_stopped/1 , status/1 , ping/1 + , send_to_remote/2 ]). -export([ get_forwards/1 @@ -171,6 +172,11 @@ ping(Pid) when is_pid(Pid) -> ping(Name) -> gen_statem:call(name(Name), ping). +send_to_remote(Pid, Msg) when is_pid(Pid) -> + gen_statem:cast(Pid, {send_to_remote, Msg}); +send_to_remote(Name, Msg) -> + gen_statem:cast(name(Name), {send_to_remote, Msg}). + %% @doc Return all forwards (local subscriptions). -spec get_forwards(id()) -> [topic()]. get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(1000)). @@ -194,7 +200,6 @@ init(#{name := Name} = ConnectOpts) -> }}. init_state(Opts) -> - IfRecordMetrics = maps:get(if_record_metrics, Opts, true), ReconnDelayMs = maps:get(reconnect_interval, Opts, ?DEFAULT_RECONNECT_DELAY_MS), StartType = maps:get(start_type, Opts, manual), Mountpoint = maps:get(forward_mountpoint, Opts, undefined), @@ -208,7 +213,6 @@ init_state(Opts) -> inflight => [], max_inflight => MaxInflightSize, connection => undefined, - if_record_metrics => IfRecordMetrics, name => Name}. open_replayq(Name, QCfg) -> @@ -321,17 +325,15 @@ common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := F {keep_state_and_data, [{reply, From, Forwards}]}; common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) -> {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]}; -common(_StateName, info, {deliver, _, Msg}, - State = #{replayq := Q, if_record_metrics := IfRecordMetric}) -> +common(_StateName, info, {deliver, _, Msg}, State = #{replayq := Q}) -> Msgs = collect([Msg]), - bridges_metrics_inc(IfRecordMetric, - 'bridge.mqtt.message_received', - length(Msgs) - ), NewQ = replayq:append(Q, Msgs), {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; common(_StateName, info, {'EXIT', _, _}, State) -> {keep_state, State}; +common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) -> + NewQ = replayq:append(Q, [Msg]), + {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}}; common(StateName, Type, Content, #{name := Name} = State) -> ?LOG(notice, "Bridge ~p discarded ~p type event at state ~p:~p", [Name, Type, StateName, Content]), @@ -401,11 +403,10 @@ do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Batch) -> do_send(#{inflight := Inflight, connection := Connection, mountpoint := Mountpoint, - connect_opts := #{forwards := Forwards}, - if_record_metrics := IfRecordMetrics} = State, QAckRef, [_ | _] = Batch) -> + connect_opts := #{forwards := Forwards}} = State, QAckRef, [_ | _] = Batch) -> Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards), ExportMsg = fun(Message) -> - bridges_metrics_inc(IfRecordMetrics, 'bridge.mqtt.message_sent'), + emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'), emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) end, ?LOG(debug, "publish to remote broker, msg: ~p, vars: ~p", [Batch, Vars]), @@ -464,6 +465,8 @@ drop_acked_batches(Q, [#{send_ack_ref := Refs, All end. +subscribe_local_topic(undefined, _Name) -> + ok; subscribe_local_topic(Topic, Name) -> do_subscribe(Topic, Name). @@ -487,7 +490,7 @@ disconnect(#{connection := Conn} = State) when Conn =/= undefined -> emqx_connector_mqtt_mod:stop(Conn), State#{connection => undefined}; disconnect(State) -> - State. + State. %% Called only when replayq needs to dump it to disk. msg_marshaller(Bin) when is_binary(Bin) -> emqx_connector_mqtt_msg:from_binary(Bin); @@ -502,20 +505,10 @@ name(Id) -> list_to_atom(str(Id)). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, - ['bridge.mqtt.message_sent', - 'bridge.mqtt.message_received' + ['bridge.mqtt.message_sent_to_remote', + 'bridge.mqtt.message_received_from_remote' ]). -bridges_metrics_inc(true, Metric) -> - emqx_metrics:inc(Metric); -bridges_metrics_inc(_IsRecordMetric, _Metric) -> - ok. - -bridges_metrics_inc(true, Metric, Value) -> - emqx_metrics:inc(Metric, Value); -bridges_metrics_inc(_IsRecordMetric, _Metric, _Value) -> - ok. - obfuscate(Map) -> maps:fold(fun(K, V, Acc) -> case is_sensitive(K) of From 085d8e8efa0edc6b72e1d779b5bcbd6f133dfcaf Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 22 Sep 2021 16:41:00 +0800 Subject: [PATCH 77/84] chore(schema): restore space --- apps/emqx/src/emqx_schema.erl | 20 +++++++++---------- apps/emqx_dashboard/src/emqx_swagger_util.erl | 13 ++++++++++++ 2 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 apps/emqx_dashboard/src/emqx_swagger_util.erl diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6c2245ad3..66db17e81 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -939,9 +939,9 @@ ssl(Defaults) -> sc(boolean(), #{ default => Df("secure_renegotiate", true) , desc => """ -SSL parameter renegotiation is a feature that allows a client and a server -to renegotiate the parameters of the SSL connection on the fly. -RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, you drop support for the insecure renegotiation, prone to MitM attacks. """ }) @@ -950,13 +950,13 @@ you drop support for the insecure renegotiation, prone to MitM attacks. sc(boolean(), #{ default => Df("client_renegotiation", true) , desc => """ -In protocols that support client-initiated renegotiation, -the cost of resources of such an operation is higher for the server than the client. -This can act as a vector for denial of service attacks. -The SSL application already takes measures to counter-act such attempts, -but client-initiated renegotiation can be strictly disabled by setting this option to false. -The default value is true. Note that disabling renegotiation can result in -long-lived connections becoming unusable due to limits on +In protocols that support client-initiated renegotiation, +the cost of resources of such an operation is higher for the server than the client. +This can act as a vector for denial of service attacks. +The SSL application already takes measures to counter-act such attempts, +but client-initiated renegotiation can be strictly disabled by setting this option to false. +The default value is true. Note that disabling renegotiation can result in +long-lived connections becoming unusable due to limits on the number of messages the underlying cipher suite can encipher. """ }) diff --git a/apps/emqx_dashboard/src/emqx_swagger_util.erl b/apps/emqx_dashboard/src/emqx_swagger_util.erl new file mode 100644 index 000000000..e2f279941 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_swagger_util.erl @@ -0,0 +1,13 @@ +%%%------------------------------------------------------------------- +%%% @author zhongwen +%%% @copyright (C) 2021, +%%% @doc +%%% +%%% @end +%%% Created : 22. 9月 2021 13:38 +%%%------------------------------------------------------------------- +-module(emqx_swagger_util). +-author("zhongwen"). + +%% API +-export([]). From da923c7df04aff0f44d1513fc0335b1aa2798b02 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 22 Sep 2021 10:39:02 +0800 Subject: [PATCH 78/84] fix(delayed): `max-delayed-messages` increase 1 than setting. --- apps/emqx_modules/src/emqx_delayed.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 823424c1f..8a8011575 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -206,9 +206,9 @@ handle_call({store, DelayedMsg = #delayed_message{key = Key}}, {reply, ok, ensure_publish_timer(Key, State)}; handle_call({store, DelayedMsg = #delayed_message{key = Key}}, - _From, State = #{max_delayed_messages := Val}) -> + _From, State = #{max_delayed_messages := Max}) -> Size = mnesia:table_info(?TAB, size), - case Size > Val of + case Size >= Max of true -> {reply, {error, max_delayed_messages_full}, State}; false -> From 5baf427b751eaaa7fbcfe0f6cb19a07cd547dfd1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 22 Sep 2021 11:08:04 +0800 Subject: [PATCH 79/84] fix(delayed): default username when get delayed message form `/publish` api. --- apps/emqx_modules/src/emqx_delayed.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 8a8011575..21757b528 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -135,7 +135,7 @@ format_delayed(Delayed) -> format_delayed(#delayed_message{key = {ExpectTimeStamp, Id}, delayed = Delayed, msg = #message{topic = Topic, from = From, - headers = #{username := Username}, + headers = Headers, qos = Qos, timestamp = PublishTimeStamp, payload = Payload}}, WithPayload) -> @@ -151,7 +151,7 @@ format_delayed(#delayed_message{key = {ExpectTimeStamp, Id}, delayed = Delayed, topic => Topic, qos => Qos, from_clientid => From, - from_username => Username + from_username => maps:get(username, Headers, undefined) }, case WithPayload of true -> From 86f47b6112f115afd0aca638404b14c19369d248 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 22 Sep 2021 22:34:46 +0200 Subject: [PATCH 80/84] chore(emqx_data_bridge): delete stale app dir --- .../src/emqx_data_bridge_schema.erl | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl deleted file mode 100644 index fe2d3947d..000000000 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ /dev/null @@ -1,25 +0,0 @@ --module(emqx_data_bridge_schema). - --export([roots/0, fields/1]). - -%%====================================================================================== -%% Hocon Schema Definitions - --define(TYPES, [mysql, pgsql, mongo, redis, ldap]). --define(BRIDGES, [hoconsc:ref(?MODULE, T) || T <- ?TYPES]). - -roots() -> ["emqx_data_bridge"]. - -fields("emqx_data_bridge") -> - [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), - default => []}}]; - -fields(mysql) -> connector_fields(emqx_connector_mysql, mysql); -fields(pgsql) -> connector_fields(emqx_connector_pgsql, pgsql); -fields(mongo) -> connector_fields(emqx_connector_mongo, mongo); -fields(redis) -> connector_fields(emqx_connector_redis, redis); -fields(ldap) -> connector_fields(emqx_connector_ldap, ldap). - -connector_fields(ConnectModule, DB) -> - [{name, hoconsc:mk(typerefl:binary())}, - {type, #{type => DB}}] ++ ConnectModule:roots(). From 79026d5900cfbbdb18196d2a12ca1f72d50990d8 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 17 Sep 2021 14:17:36 +0800 Subject: [PATCH 81/84] chore(authn): add supervisor for authn and fix checking errors --- apps/emqx/src/emqx_authentication.erl | 3 +- apps/emqx/src/emqx_authentication_sup.erl | 48 +++++++++++++++++++ apps/emqx/src/emqx_broker_sup.erl | 16 +++---- apps/emqx_authn/src/emqx_authn_api.erl | 2 +- .../src/simple_authn/emqx_authn_http.erl | 2 - .../src/simple_authn/emqx_authn_mongodb.erl | 3 -- .../src/simple_authn/emqx_authn_mysql.erl | 1 - .../src/simple_authn/emqx_authn_pgsql.erl | 1 - .../src/simple_authn/emqx_authn_redis.erl | 1 - 9 files changed, 58 insertions(+), 19 deletions(-) create mode 100644 apps/emqx/src/emqx_authentication_sup.erl diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index bcb38471a..4200190ac 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -281,8 +281,7 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} check_config(Config) -> #{authentication := CheckedConfig} = - hocon_schema:check_plain(?MODULE, #{<<"authentication">> => Config}, - #{nullable => true, atom_key => true}), + hocon_schema:check_plain(?MODULE, #{<<"authentication">> => Config}, #{atom_key => true}), CheckedConfig. %%------------------------------------------------------------------------------ diff --git a/apps/emqx/src/emqx_authentication_sup.erl b/apps/emqx/src/emqx_authentication_sup.erl new file mode 100644 index 000000000..cdae04f3c --- /dev/null +++ b/apps/emqx/src/emqx_authentication_sup.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 100, + period => 10}, + + AuthN = #{id => emqx_authentication, + start => {emqx_authentication, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [emqx_authentication]}, + + {ok, {SupFlags, [AuthN]}}. \ No newline at end of file diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index a479e9ff1..761537e57 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -44,13 +44,13 @@ init([]) -> modules => [emqx_shared_sub]}, %% Authentication - AuthN = #{id => authn, - start => {emqx_authentication, start_link, []}, - restart => permanent, - shutdown => 2000, - type => worker, - modules => [emqx_authentication]}, - + AuthNSup = #{id => emqx_authentication_sup, + start => {emqx_authentication_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_authentication_sup]}, + %% Broker helper Helper = #{id => helper, start => {emqx_broker_helper, start_link, []}, @@ -59,5 +59,5 @@ init([]) -> type => worker, modules => [emqx_broker_helper]}, - {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, AuthN, Helper]}}. + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, AuthNSup, Helper]}}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index b58a2a214..540cf86e3 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1971,7 +1971,7 @@ find_config(AuthenticatorID, AuthenticatorsConfig) -> fill_defaults(Config) -> #{<<"authentication">> := CheckedConfig} = hocon_schema:check_plain( - ?AUTHN, #{<<"authentication">> => Config}, #{nullable => true, no_conversion => true}), + ?AUTHN, #{<<"authentication">> => Config}, #{no_conversion => true}), CheckedConfig. convert_certs(#{<<"ssl">> := SSLOpts} = Config) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9814d3e58..2fa29d2df 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -77,7 +77,6 @@ validations() -> ]. url(type) -> binary(); -url(nullable) -> false; url(validator) -> [fun check_url/1]; url(_) -> undefined. @@ -98,7 +97,6 @@ headers_no_content_type(default) -> default_headers_no_content_type(); headers_no_content_type(_) -> undefined. body(type) -> map(); -body(nullable) -> false; body(validator) -> [fun check_body/1]; body(_) -> undefined. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 9d77c673c..5ad148009 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -70,15 +70,12 @@ common_fields() -> ] ++ emqx_authn_schema:common_fields(). collection(type) -> binary(); -collection(nullable) -> false; collection(_) -> undefined. selector(type) -> map(); -selector(nullable) -> false; selector(_) -> undefined. password_hash_field(type) -> binary(); -password_hash_field(nullable) -> false; password_hash_field(_) -> undefined. salt_field(type) -> binary(); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 991bb6aee..87c61da1e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -63,7 +63,6 @@ salt_position(default) -> prefix; salt_position(_) -> undefined. query(type) -> string(); -query(nullable) -> false; query(_) -> undefined. query_timeout(type) -> integer(); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index c497074de..940c50519 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -59,7 +59,6 @@ password_hash_algorithm(default) -> sha256; password_hash_algorithm(_) -> undefined. query(type) -> string(); -query(nullable) -> false; query(_) -> undefined. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 949aeeaea..5926740a8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -66,7 +66,6 @@ common_fields() -> ] ++ emqx_authn_schema:common_fields(). query(type) -> string(); -query(nullable) -> false; query(_) -> undefined. password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; From 48600492680284d1cfef0c16df22ffb44df392af Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 17 Sep 2021 15:29:02 +0800 Subject: [PATCH 82/84] chore(authn): insert final newline --- apps/emqx/src/emqx_authentication_sup.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_authentication_sup.erl b/apps/emqx/src/emqx_authentication_sup.erl index cdae04f3c..7b0d77f71 100644 --- a/apps/emqx/src/emqx_authentication_sup.erl +++ b/apps/emqx/src/emqx_authentication_sup.erl @@ -45,4 +45,4 @@ init([]) -> type => worker, modules => [emqx_authentication]}, - {ok, {SupFlags, [AuthN]}}. \ No newline at end of file + {ok, {SupFlags, [AuthN]}}. From a99e54a6a2da10e623e3b987f8b7e91980185ad7 Mon Sep 17 00:00:00 2001 From: Paulus Lucas Date: Thu, 23 Sep 2021 05:11:45 +0200 Subject: [PATCH 83/84] feat(helm): add envFromSecret (#5778) * (feature): Adding EnvFrom a specified file * (update): Update values.yaml with envFrom * (doc) : update readme with envFromSecret --- deploy/charts/emqx/README.md | 1 + deploy/charts/emqx/templates/StatefulSet.yaml | 4 ++++ deploy/charts/emqx/values.yaml | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index d49ee692d..0e66ca2c9 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -37,6 +37,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `image.repository` | EMQ X Image name |emqx/emqx| | `image.pullPolicy` | The image pull policy |IfNotPresent| | `image.pullSecrets ` | The image pull secrets |`[]` (does not add image pull secrets to deployed pods)| +| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | | `persistence.enabled` | Enable EMQX persistence using PVC |false| | `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)| diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 24e805f76..dc2a76b68 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -114,6 +114,10 @@ spec: envFrom: - configMapRef: name: {{ include "emqx.fullname" . }}-env + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ .Values.envFromSecret }} + {{- end }} env: - name: EMQX_NAME value: {{ .Release.Name }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 72c8265c6..327c0960b 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -14,6 +14,13 @@ image: # pullSecrets: # - myRegistryKeySecretName + +# The name of a secret in the same kubernetes namespace which contains values to +# be added to the environment (must be manually created) +# This can be useful for passwords and logins, etc. + +# envFromSecret: "emqx-secrets" + ## Forces the recreation of pods during helm upgrades. This can be useful to update configuration values even if the container image did not change. recreatePods: false From 29b96bae2e1e58af6a95dbdfe913d02059028a10 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 23 Sep 2021 11:25:02 +0800 Subject: [PATCH 84/84] fix(dashboard): create user return 405 --- apps/emqx_dashboard/src/emqx_dashboard_api.erl | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 874d2bf2b..5b22facf6 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -100,6 +100,15 @@ schema("/users") -> 200 => mk(array(ref(?MODULE, user)), #{desc => "User lists"}) } + }, + post => #{ + tags => [<<"dashboard">>], + description => <<"Create dashboard users">>, + requestBody => fields(user_password), + responses => #{ + 200 => <<"Create user successfully">>, + 400 => [{code, mk(string(), #{example => 'CREATE_FAIL'})}, + {message, mk(string(), #{example => "Create user failed"})}]} } }; @@ -149,12 +158,14 @@ schema("/users/:username/change_pwd") -> fields(user) -> [ {tag, - mk(string(), + mk(binary(), #{desc => <<"tag">>, example => "administrator"})}, {username, - mk(string(), + mk(binary(), #{desc => <<"username">>, example => "emqx"})} - ]. + ]; +fields(user_password) -> + fields(user) ++ [{password, mk(binary(), #{desc => "Password"})}]. login(post, #{body := Params}) -> Username = maps:get(<<"username">>, Params),