From 7f349d814e320f2f323e2cf1586fc3597f273935 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 3 Feb 2021 17:03:16 +0800 Subject: [PATCH 01/12] test(ci): unify the pgsql confs --- .ci/compatibility_tests/docker-compose-pgsql-tls.yaml | 6 +++--- .github/workflows/run_cts_tests.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml index d3d9d93b5..6bb3d321e 100644 --- a/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml +++ b/.ci/compatibility_tests/docker-compose-pgsql-tls.yaml @@ -24,9 +24,9 @@ services: image: emqx_pgsql:${PGSQL_TAG} restart: always environment: - POSTGRES_DB: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_DB: mqtt + POSTGRES_USER: root + POSTGRES_PASSWORD: public ports: - "5432:5432" command: diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index fdca76a7c..5e0335d92 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -200,9 +200,9 @@ jobs: sed -i 's|^[#[:space:]]*auth.pgsql.ssl.tls_versions[ \t]*=.*|auth.pgsql.ssl.tls_versions = tlsv1.2,tlsv1.1|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf fi - sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup From c3642c5c831e0f0f6c17dc3ab1856cb7cb3d4d60 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 4 Feb 2021 15:29:00 +0800 Subject: [PATCH 02/12] refactor(pgsql): set the default ssl version to tlsv1.3,tlsv1.2,tlsv1.1 --- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 14 +++++++------- apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index ef8e7533a..d27956b16 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -22,7 +22,7 @@ auth.pgsql.username = root ## PostgreSQL password. ## ## Value: String -# auth.pgsql.password = +#auth.pgsql.password = ## PostgreSQL database. ## @@ -39,13 +39,13 @@ auth.pgsql.encoding = utf8 ## Value: on | off auth.pgsql.ssl = off -## TLS version -## You can configure multi-version use "," split, -## default value is :tlsv1.2 -## Example: -## tlsv1.1,tlsv1.2,tlsv1.3 +## TLS version. ## -#auth.pgsql.ssl.tls_versions = tlsv1.2 +## Available enum values: +## tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +## +## Value: String, seperated by ',' +#auth.pgsql.ssl.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1 ## SSL keyfile. ## diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index 859495a60..77a239ba9 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -36,7 +36,7 @@ ]}. {mapping, "auth.pgsql.ssl.tls_versions", "emqx_auth_pgsql.server", [ - {default, "tlsv1.2"}, + {default, "tlsv1.3,tlsv1.2,tlsv1.1"}, {datatype, string} ]}. @@ -92,9 +92,9 @@ SslOpts = fun(Prefix) -> Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined), + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, {versions, [list_to_existing_atom(Value) - ||Value <- string:tokens(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf), " ,")]}}]) + || Value <- string:tokens(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf), " ,")]}]) end, %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 From daa9648db5964ab89b4350b4e6f459833e654903 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 4 Feb 2021 15:30:25 +0800 Subject: [PATCH 03/12] chore(ci): delete the ssl version configuration --- .github/workflows/run_cts_tests.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 5e0335d92..f5ad55183 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -194,12 +194,6 @@ jobs: run: | docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d - if [ "$PGSQL_TAG" = "12" ] || [ "$PGSQL_TAG" = "13" ]; then - sed -i 's|^[#[:space:]]*auth.pgsql.ssl.tls_versions[ \t]*=.*|auth.pgsql.ssl.tls_versions = tlsv1.3,tlsv1.2|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - else - sed -i 's|^[#[:space:]]*auth.pgsql.ssl.tls_versions[ \t]*=.*|auth.pgsql.ssl.tls_versions = tlsv1.2,tlsv1.1|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - fi - sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf From 2852ac79d4759111fa55ab07e31257ac06e115db Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Feb 2021 20:37:54 +0100 Subject: [PATCH 04/12] feat(tls): Add a tls lib module emqx_tls_lib is added to help ensure integrity of tls version and cipher options. --- src/emqx_tls_lib.erl | 98 +++++++++++++++++++++++++++++++++++++ test/emqx_tls_lib_tests.erl | 64 ++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 src/emqx_tls_lib.erl create mode 100644 test/emqx_tls_lib_tests.erl diff --git a/src/emqx_tls_lib.erl b/src/emqx_tls_lib.erl new file mode 100644 index 000000000..d78331af0 --- /dev/null +++ b/src/emqx_tls_lib.erl @@ -0,0 +1,98 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_tls_lib). + +-export([ default_versions/0 + , integral_versions/1 + , default_ciphers/0 + , default_ciphers/1 + , integral_ciphers/2 + ]). + +-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso is_list(hd(L)))). + +%% @doc Returns the default supported tls versions. +-spec default_versions() -> [atom()]. +default_versions() -> + OtpRelease = list_to_integer(erlang:system_info(otp_release)), + integral_versions(default_versions(OtpRelease)). + +%% @doc Validate a given list of desired tls versions. +%% raise an error exception if non of them are available. +-spec integral_versions([ssl:tls_version()]) -> [ssl:tls_version()]. +integral_versions(Desired) -> + {_, Available} = lists:keyfind(available, 1, ssl:versions()), + case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of + [] -> erlang:error(#{ reason => no_available_tls_version + , desired => Desired + , available => Available + }); + Filtered -> + Filtered + end. + +%% @doc Return a list of default (openssl string format) cipher suites. +-spec default_ciphers() -> [string()]. +default_ciphers() -> default_ciphers(tls_versions()). + +%% @doc Return a list of (openssl string format) cipher suites. +-spec default_ciphers([ssl:tls_version()]) -> [string()]. +default_ciphers(['tlsv1.3']) -> + %% When it's only tlsv1.3 wanted, use 'exclusive' here + %% because 'all' returns legacy cipher suites too, + %% which does not make sense since tlsv1.3 can not use + %% legacy cipher suites. + ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); +default_ciphers(Versions) -> + %% assert non-empty + [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])). + +%% @doc Ensure version & cipher-suites integrity. +-spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. +integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined -> + %% not configured + integral_ciphers(Versions, default_ciphers(Versions)); +integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) -> + %% ensure tlsv1.3 ciphers if none of them is found in Ciphers + dedup(ensure_tls1_3_cipher(lists:member('tlsv1.3', Versions), Ciphers)); +integral_ciphers(Versions, Ciphers) when is_binary(Ciphers) -> + %% parse binary + integral_ciphers(Versions, binary_to_list(Ciphers)); +integral_ciphers(Versions, Ciphers) -> + %% parse comma separated cipher suite names + integral_ciphers(Versions, string:tokens(Ciphers, ", ")). + +%% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user +%% did not provide it from config --- which is a common mistake +ensure_tls1_3_cipher(true, Ciphers) -> + Tls13Ciphers = default_ciphers(['tlsv1.3']), + case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of + true -> Ciphers; + false -> Tls13Ciphers ++ Ciphers + end; +ensure_tls1_3_cipher(false, Ciphers) -> + Ciphers. + +%% tlsv1.3 is available from OTP-22 but we do not want to use until 23. +default_versions(OtpRelease) when OtpRelease >= 23 -> + ['tlsv1.3' | default_tls_versions(22)]; +default_versions(_) -> + ['tlsv1.2','tlsv1.1', tlsv1]. + +%% Deduplicate a list without re-ordering the elements. +dedup([]) -> []; +dedup([H | T]) -> [H | dedup([I || I <- T, I =/= H])]. diff --git a/test/emqx_tls_lib_tests.erl b/test/emqx_tls_lib_tests.erl new file mode 100644 index 000000000..452909db2 --- /dev/null +++ b/test/emqx_tls_lib_tests.erl @@ -0,0 +1,64 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_tls_lib_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%% one of the cipher suite from tlsv1.2 and tlsv1.3 each +-define(TLS_12_CIPHER, "ECDHE-ECDSA-AES256-GCM-SHA384"). +-define(TLS_13_CIPHER, "TLS_AES_256_GCM_SHA384"). + +ensure_tls13_ciphers_added_test() -> + Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3'], [?TLS_12_CIPHER]), + ?assert(lists:member(?TLS_12_CIPHER, Ciphers)), + ?assert(lists:member(?TLS_13_CIPHER, Ciphers)). + +legacy_cipher_suites_test() -> + Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.2'], [?TLS_12_CIPHER]), + ?assertEqual([?TLS_12_CIPHER], Ciphers). + +use_default_ciphers_test() -> + Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3', 'tlsv1.2'], ""), + ?assert(lists:member(?TLS_12_CIPHER, Ciphers)), + ?assert(lists:member(?TLS_13_CIPHER, Ciphers)). + +ciphers_format_test_() -> + String = ?TLS_13_CIPHER ++ "," ++ ?TLS_12_CIPHER, + Binary = iolist_to_binary(String), + List = [?TLS_13_CIPHER, ?TLS_12_CIPHER], + [ {"string", fun() -> test_cipher_format(String) end} + , {"binary", fun() -> test_cipher_format(Binary) end} + , {"string-list", fun() -> test_cipher_format(List) end} + ]. + +test_cipher_format(Input) -> + Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3', 'tlsv1.2'], Input), + ?assertEqual([?TLS_13_CIPHER, ?TLS_12_CIPHER], Ciphers). + +tls_versions_test() -> + ?assert(lists:member('tlsv1.3', emqx_tls_lib:default_versions())). + +tls_version_unknown_test() -> + ?assertError(#{reason := no_available_tls_version}, + emqx_tls_lib:integral_versions([])), + ?assertError(#{reason := no_available_tls_version}, + emqx_tls_lib:integral_versions([foo])). + +cipher_suites_no_duplication_test() -> + AllCiphers = emqx_tls_lib:default_ciphers(), + ?assertEqual(length(AllCiphers), length(lists:usort(AllCiphers))). + From f000b6583c32de333bc7a9ffa647c2ddcc01a07c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Feb 2021 10:49:35 +0100 Subject: [PATCH 05/12] fix(tls): Ensure tls config integrity For default tsl version and ciphers, we try to use otp release number to determin if we want to use tlsv1.3 For default configs, we try to porivde both tlsv1.3 and ciphers in config (even for commented out configs) --- .../emqx_auth_http/src/emqx_auth_http_app.erl | 9 ++++---- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 2 +- apps/emqx_bridge_mqtt/README.md | 2 +- apps/emqx_bridge_mqtt/docs/guide.rst | 8 +++---- .../etc/emqx_bridge_mqtt.conf | 1 + .../priv/emqx_bridge_mqtt.schema | 2 +- .../src/emqx_bridge_mqtt_actions.erl | 21 ++++++++----------- apps/emqx_coap/priv/emqx_coap.schema | 5 +---- apps/emqx_dashboard/etc/emqx_dashboard.conf | 3 ++- apps/emqx_exproto/test/emqx_exproto_SUITE.erl | 7 ++----- apps/emqx_management/etc/emqx_management.conf | 3 ++- apps/emqx_stomp/etc/emqx_stomp.conf | 3 ++- .../src/emqx_web_hook_actions.erl | 13 ++++++------ apps/emqx_web_hook/src/emqx_web_hook_app.erl | 13 ++++++------ etc/emqx.conf | 8 ++++--- src/emqx_tls_lib.erl | 12 +++++------ 16 files changed, 53 insertions(+), 59 deletions(-) diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index ba88ca3be..9b6b265d4 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -74,11 +74,10 @@ translate_env(EnvName) -> (_) -> true end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), - TlsVers = ['tlsv1.2','tlsv1.1',tlsv1], - NTLSOpts = [{versions, TlsVers}, - {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> - Ciphers ++ ssl:cipher_suites(all, TlsVer) - end, [], TlsVers)} | TLSOpts], + NTLSOpts = [ {versions, emqx_tls_lib:default_versions()} + , {ciphers, emqx_tls_lib:default_ciphers()} + | TLSOpts + ], [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] end, PoolOpts = [{host, Host}, diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index ef8e7533a..b2eec355b 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -43,7 +43,7 @@ auth.pgsql.ssl = off ## You can configure multi-version use "," split, ## default value is :tlsv1.2 ## Example: -## tlsv1.1,tlsv1.2,tlsv1.3 +## tlsv1.2,tlsv1.1 ## #auth.pgsql.ssl.tls_versions = tlsv1.2 diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md index 6656aa36f..456fae584 100644 --- a/apps/emqx_bridge_mqtt/README.md +++ b/apps/emqx_bridge_mqtt/README.md @@ -126,7 +126,7 @@ bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-S bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version -bridge.mqtt.emqx2.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +bridge.mqtt.emqx2.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## Forwarding topics of the message bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst index 73350ca1f..47235aa7d 100644 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ b/apps/emqx_bridge_mqtt/docs/guide.rst @@ -133,9 +133,6 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co ## Key file of Client SSL connection bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem - ## SSL encryption - bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 - ## TTLS PSK password ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time ## @@ -146,7 +143,10 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version - bridge.mqtt.emqx2.tls_versions = tlsv1.2,tlsv1.1,tlsv1 + bridge.mqtt.emqx2.tls_versions = tlsv1.2 + + ## SSL encryption + bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 ## Forwarding topics of the message bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index ec5656af4..db59270c0 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -128,6 +128,7 @@ bridge.mqtt.aws.keepalive = 60s ## TLS versions used by the bridge. ## +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier ## Value: String bridge.mqtt.aws.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 diff --git a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema index 301737afd..3168bfc14 100644 --- a/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema +++ b/apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema @@ -90,7 +90,7 @@ {mapping, "bridge.mqtt.$name.tls_versions", "emqx_bridge_mqtt.bridges", [ {datatype, string}, - {default, "tlsv1,tlsv1.1,tlsv1.2"} + {default, "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1"} ]}. {mapping, "bridge.mqtt.$name.reconnect_interval", "emqx_bridge_mqtt.bridges", [ diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index d63f20141..87bc6c694 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -671,12 +671,6 @@ format_data([], Msg) -> format_data(Tokens, Msg) -> emqx_rule_utils:proc_tmpl(Tokens, Msg). -tls_versions() -> - ['tlsv1.2','tlsv1.1', tlsv1]. - -ciphers(Ciphers) -> - string:tokens(str(Ciphers), ", "). - subscriptions(Subscriptions) -> scan_binary(<<"[", Subscriptions/binary, "].">>). @@ -749,6 +743,8 @@ options(Options, PoolName) -> Topic -> [{subscriptions, [{Topic, Get(<<"qos">>)}]} | Subscriptions] end, + %% TODO check why only ciphers are configurable but not versions + TlsVersions = emqx_tls_lib:default_versions(), [{address, binary_to_list(Address)}, {bridge_mode, GetD(<<"bridge_mode">>, true)}, {clean_start, true}, @@ -761,12 +757,13 @@ options(Options, PoolName) -> {proto_ver, mqtt_ver(Get(<<"proto_ver">>))}, {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)}, {ssl, cuttlefish_flag:parse(str(Get(<<"ssl">>)))}, - {ssl_opts, [{versions, tls_versions()}, - {ciphers, ciphers(Get(<<"ciphers">>))}, - {keyfile, str(Get(<<"keyfile">>))}, - {certfile, str(Get(<<"certfile">>))}, - {cacertfile, str(Get(<<"cacertfile">>))} - ]}] ++ Subscriptions1 + {ssl_opts, [ {keyfile, str(Get(<<"keyfile">>))} + , {certfile, str(Get(<<"certfile">>))} + , {cacertfile, str(Get(<<"cacertfile">>))} + , {versions, TlsVersions} + , {ciphers, emqx_tls_lib:integral_ciphers(TlsVersions, Get(<<"ciphers">>))} + ]} + ] ++ Subscriptions1 end. diff --git a/apps/emqx_coap/priv/emqx_coap.schema b/apps/emqx_coap/priv/emqx_coap.schema index 465979964..da367f098 100644 --- a/apps/emqx_coap/priv/emqx_coap.schema +++ b/apps/emqx_coap/priv/emqx_coap.schema @@ -75,10 +75,7 @@ end}. Ciphers = case cuttlefish:conf_get("coap.dtls.ciphers", Conf, undefined) of undefined -> - lists:foldl( - fun(TlsVer, Ciphers) -> - Ciphers ++ ssl:cipher_suites(all, TlsVer) - end, [], ['dtlsv1', 'dtlsv1.2']); + lists:append([ssl:cipher_suites(all, V, openssl) || V <- ['dtlsv1.2', 'dtlsv1']]); C -> SplitFun(C) end, diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 21dbbd04a..933b9885d 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -105,7 +105,8 @@ dashboard.listener.http.ipv6_v6only = false ## TLS versions only to protect from POODLE attack. ## ## Value: String, seperated by ',' -## dashboard.listener.https.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## dashboard.listener.https.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## See: 'listener.ssl..ciphers' in emq.conf ## diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index 2ad79b77d..c5cabe154 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -425,8 +425,8 @@ udp_opts() -> ssl_opts() -> Certs = certs("key.pem", "cert.pem", "cacert.pem"), - [{versions, ['tlsv1.2','tlsv1.1',tlsv1]}, - {ciphers, ciphers('tlsv1.2')}, + [{versions, emqx_tls_lib:default_versions()}, + {ciphers, emqx_tls_lib:default_ciphers()}, {verify, verify_peer}, {fail_if_no_peer_cert, true}, {secure_renegotiate, false}, @@ -437,9 +437,6 @@ dtls_opts() -> Opts = ssl_opts(), lists:keyreplace(versions, 1, Opts, {versions, ['dtlsv1.2', 'dtlsv1']}). -ciphers(Version) -> - proplists:get_value(ciphers, emqx_ct_helpers:client_ssl(Version)). - %%-------------------------------------------------------------------- %% Client-Opts diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index 4bb3f83dc..aa737add9 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -45,7 +45,8 @@ management.listener.http.ipv6_v6only = false ## management.listener.https.keyfile = etc/certs/key.pem ## management.listener.https.cacertfile = etc/certs/cacert.pem ## management.listener.https.verify = verify_peer -## management.listener.https.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## management.listener.https.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## management.listener.https.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA ## management.listener.https.fail_if_no_peer_cert = true ## management.listener.https.inet6 = false diff --git a/apps/emqx_stomp/etc/emqx_stomp.conf b/apps/emqx_stomp/etc/emqx_stomp.conf index def48fbc8..b404e99ee 100644 --- a/apps/emqx_stomp/etc/emqx_stomp.conf +++ b/apps/emqx_stomp/etc/emqx_stomp.conf @@ -58,7 +58,8 @@ stomp.listener.max_connections = 512 ## TLS versions only to protect from POODLE attack. ## ## Value: String, seperated by ',' -## stomp.listener.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## stomp.listener.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## SSL Handshake timeout. ## diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index bcd29bd28..f5d27f09c 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -354,12 +354,11 @@ pool_opts(Params = #{<<"url">> := URL}) -> (_) -> true end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), - TlsVers = ['tlsv1.2', 'tlsv1.1', tlsv1], - NTLSOpts = [{verify, VerifyType}, - {versions, TlsVers}, - {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> - Ciphers ++ ssl:cipher_suites(all, TlsVer) - end, [], TlsVers)} | TLSOpts], + NTLSOpts = [ {verify, VerifyType} + , {versions, emqx_tls_lib:default_versions()} + , {ciphers, emqx_tls_lib:default_ciphers()} + | TLSOpts + ], [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] end, [{host, Host}, @@ -397,4 +396,4 @@ test_http_connect(Conf) -> Err:Reason:ST -> ?LOG(error, "check http_connectivity failed: ~p, ~0p", [Conf, {Err, Reason, ST}]), false - end. \ No newline at end of file + end. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index c8083e195..d31df9b93 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -75,12 +75,11 @@ translate_env() -> TLSOpts = lists:filter(fun({_K, V}) -> V /= <<>> andalso V /= undefined andalso V /= "" andalso true end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]), - TlsVers = ['tlsv1.2','tlsv1.1',tlsv1], - NTLSOpts = [{verify, VerifyType}, - {versions, TlsVers}, - {ciphers, lists:foldl(fun(TlsVer, Ciphers) -> - Ciphers ++ ssl:cipher_suites(all, TlsVer) - end, [], TlsVers)} | TLSOpts], + NTLSOpts = [ {verify, VerifyType} + , {versions, emqx_tls_lib:default_versions()} + , {ciphers, emqx_tls_lib:default_ciphers()} + | TLSOpts + ], [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}] end, PoolOpts = [{host, Host}, @@ -114,4 +113,4 @@ parse_host(Host) -> {ok, _} -> {inet6, Host}; {error, _} -> {inet, Host} end - end. \ No newline at end of file + end. diff --git a/etc/emqx.conf b/etc/emqx.conf index cb8c27b16..33db7abb6 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1317,7 +1317,8 @@ listener.ssl.external.access.1 = allow all ## See: http://erlang.org/doc/man/ssl.html ## ## Value: String, seperated by ',' -## listener.ssl.external.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## listener.ssl.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## TLS Handshake timeout. ## @@ -1785,7 +1786,7 @@ listener.wss.external.access.1 = allow all ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.ws.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.wss.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## Enable the Proxy Protocol V1/2 support. ## @@ -1806,7 +1807,8 @@ listener.wss.external.access.1 = allow all ## See: listener.ssl.$name.tls_versions ## ## Value: String, seperated by ',' -## listener.wss.external.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## listener.wss.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 ## Path to the file containing the user's private PEM-encoded key. ## diff --git a/src/emqx_tls_lib.erl b/src/emqx_tls_lib.erl index d78331af0..6153160e7 100644 --- a/src/emqx_tls_lib.erl +++ b/src/emqx_tls_lib.erl @@ -47,7 +47,7 @@ integral_versions(Desired) -> %% @doc Return a list of default (openssl string format) cipher suites. -spec default_ciphers() -> [string()]. -default_ciphers() -> default_ciphers(tls_versions()). +default_ciphers() -> default_ciphers(default_versions()). %% @doc Return a list of (openssl string format) cipher suites. -spec default_ciphers([ssl:tls_version()]) -> [string()]. @@ -68,7 +68,7 @@ integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undef integral_ciphers(Versions, default_ciphers(Versions)); integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) -> %% ensure tlsv1.3 ciphers if none of them is found in Ciphers - dedup(ensure_tls1_3_cipher(lists:member('tlsv1.3', Versions), Ciphers)); + dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers)); integral_ciphers(Versions, Ciphers) when is_binary(Ciphers) -> %% parse binary integral_ciphers(Versions, binary_to_list(Ciphers)); @@ -78,20 +78,20 @@ integral_ciphers(Versions, Ciphers) -> %% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user %% did not provide it from config --- which is a common mistake -ensure_tls1_3_cipher(true, Ciphers) -> +ensure_tls13_cipher(true, Ciphers) -> Tls13Ciphers = default_ciphers(['tlsv1.3']), case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of true -> Ciphers; false -> Tls13Ciphers ++ Ciphers end; -ensure_tls1_3_cipher(false, Ciphers) -> +ensure_tls13_cipher(false, Ciphers) -> Ciphers. %% tlsv1.3 is available from OTP-22 but we do not want to use until 23. default_versions(OtpRelease) when OtpRelease >= 23 -> - ['tlsv1.3' | default_tls_versions(22)]; + ['tlsv1.3' | default_versions(22)]; default_versions(_) -> - ['tlsv1.2','tlsv1.1', tlsv1]. + ['tlsv1.2', 'tlsv1.1', tlsv1]. %% Deduplicate a list without re-ordering the elements. dedup([]) -> []; From 9cae8784c06948867194fa3e27a790ff4c06f066 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Feb 2021 23:10:47 +0100 Subject: [PATCH 06/12] refactor(emqx_modules): Move modules config to app dir --- .../test/props/prop_webhook_confs.erl | 1 - etc/emqx.conf | 58 ------------ .../emqx_modules/etc/emqx_modules.conf | 58 +++++++++++- .../emqx_modules/priv/emqx_modules.schema | 90 ++++++++++++++++++- .../src/emqx_mod_acl_internal.erl | 8 +- .../emqx_modules/src/emqx_modules.erl | 20 +++-- .../emqx_modules/src/emqx_modules_app.erl | 15 ++++ .../emqx_modules/test/emqx_modules_SUITE.erl | 12 +-- .../emqx_modules_SUITE_data}/loaded_modules | 0 priv/emqx.schema | 87 ------------------ test/emqx_acl_cache_SUITE.erl | 23 +---- test/mqtt_protocol_v5_SUITE.erl | 7 +- 12 files changed, 190 insertions(+), 189 deletions(-) rename {test/emqx_SUITE_data => lib-opensource/emqx_modules/test/emqx_modules_SUITE_data}/loaded_modules (100%) diff --git a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl index 24903ddec..8a637c720 100644 --- a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl +++ b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl @@ -62,7 +62,6 @@ do_teardown(_) -> set_special_cfgs(_) -> application:set_env(emqx, plugins_loaded_file, undefined), - application:set_env(emqx, modules_loaded_file, undefined), ok. assert_confs([{"web.hook.api.url", Url}|More], Envs) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index cb8c27b16..d6340f798 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2025,64 +2025,6 @@ listener.wss.external.allow_origin_absence = true ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 -##-------------------------------------------------------------------- -## Modules -##-------------------------------------------------------------------- -## The file to store loaded module names. -## -## Value: File -modules.loaded_file = {{ platform_data_dir }}/loaded_modules - -##-------------------------------------------------------------------- -## Presence Module - -## Sets the QoS for presence MQTT message. -## -## Value: 0 | 1 | 2 -module.presence.qos = 1 - -##-------------------------------------------------------------------- -## Subscription Module - -## Subscribe the Topics automatically when client connected. -## -## Value: String -## module.subscription.1.topic = connected/%c/%u - -## Qos of the proxy subscription. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.qos = 0 - -## No Local of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.nl = 0 - -## Retain As Published of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.rap = 0 - -## Retain Handling of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.rh = 0 - -##-------------------------------------------------------------------- -## Rewrite Module - -## {rewrite, Topic, Re, Dest} -## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 -## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 - ##------------------------------------------------------------------- ## Plugins ##------------------------------------------------------------------- diff --git a/lib-opensource/emqx_modules/etc/emqx_modules.conf b/lib-opensource/emqx_modules/etc/emqx_modules.conf index 1bb8bf6d7..218d86234 100644 --- a/lib-opensource/emqx_modules/etc/emqx_modules.conf +++ b/lib-opensource/emqx_modules/etc/emqx_modules.conf @@ -1 +1,57 @@ -# empty +##-------------------------------------------------------------------- +## Modules +##-------------------------------------------------------------------- +## The file to store loaded module names. +## +## Value: File +modules.loaded_file = {{ platform_data_dir }}/loaded_modules + +##-------------------------------------------------------------------- +## Presence Module + +## Sets the QoS for presence MQTT message. +## +## Value: 0 | 1 | 2 +module.presence.qos = 1 + +##-------------------------------------------------------------------- +## Subscription Module + +## Subscribe the Topics automatically when client connected. +## +## Value: String +## module.subscription.1.topic = connected/%c/%u + +## Qos of the proxy subscription. +## +## Value: 0 | 1 | 2 +## Default: 0 +## module.subscription.1.qos = 0 + +## No Local of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 +## Default: 0 +## module.subscription.1.nl = 0 + +## Retain As Published of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 +## Default: 0 +## module.subscription.1.rap = 0 + +## Retain Handling of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 | 2 +## Default: 0 +## module.subscription.1.rh = 0 + +##-------------------------------------------------------------------- +## Rewrite Module + +## {rewrite, Topic, Re, Dest} +## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 +## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 diff --git a/lib-opensource/emqx_modules/priv/emqx_modules.schema b/lib-opensource/emqx_modules/priv/emqx_modules.schema index d7c52c644..55606754c 100644 --- a/lib-opensource/emqx_modules/priv/emqx_modules.schema +++ b/lib-opensource/emqx_modules/priv/emqx_modules.schema @@ -1 +1,89 @@ -% empty +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +{mapping, "modules.loaded_file", "emqx_modules.modules_loaded_file", [ + {datatype, string} +]}. + +{mapping, "module.presence.qos", "emqx_modules.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.topic", "emqx_modules.modules", [ + {datatype, string} +]}. + +{mapping, "module.subscription.$id.qos", "emqx_modules.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.nl", "emqx_modules.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rap", "emqx_modules.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rh", "emqx_modules.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.rewrite.rule.$id", "emqx_modules.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.pub.rule.$id", "emqx_modules.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.sub.rule.$id", "emqx_modules.modules", [ + {datatype, string} +]}. + +{translation, "emqx_modules.modules", fun(Conf, _, Conf1) -> + Subscriptions = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), + TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], + [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), + PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), + SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), + TotalRules = lists:append( + [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, + [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules + ), + lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + %% TODO: acl_file config should be moved to emqx_modules.conf + %% when all the plubin tests stops using it in the old way. + [{emqx_mod_acl_internal, [{acl_file, {emqx, get_env, [acl_file]}}]}] + %[{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] + ]) +end}. diff --git a/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl b/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl index 1701de69d..2d347151d 100644 --- a/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl +++ b/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl @@ -43,7 +43,13 @@ %%-------------------------------------------------------------------- load(Env) -> - Rules = rules_from_file(proplists:get_value(acl_file, Env)), + %% TODO: acl_file config should be moved to emqx_modules.conf + %% when all the plubin tests stops using it in the old way. + File = case proplists:get_value(acl_file, Env) of + {emqx, get_env, _} -> emqx:get_env(acl_file); + F -> F + end, + Rules = rules_from_file(File), emqx_hooks:add('client.check_acl', {?MODULE, check_acl, [Rules]}, -1). unload(_Env) -> diff --git a/lib-opensource/emqx_modules/src/emqx_modules.erl b/lib-opensource/emqx_modules/src/emqx_modules.erl index ffcc456f0..18c6fb856 100644 --- a/lib-opensource/emqx_modules/src/emqx_modules.erl +++ b/lib-opensource/emqx_modules/src/emqx_modules.erl @@ -30,6 +30,8 @@ , load_module/2 ]). +-define(APP, ?MODULE). + %% @doc List all available plugins -spec(list() -> [{atom(), boolean()}]). list() -> @@ -38,7 +40,7 @@ list() -> %% @doc Load all the extended modules. -spec(load() -> ok). load() -> - case emqx:get_env(modules_loaded_file) of + case get_env(modules_loaded_file) of undefined -> ok; File -> load_modules(File) @@ -59,7 +61,7 @@ load(ModuleName) -> %% @doc Unload all the extended modules. -spec(unload() -> ok). unload() -> - case emqx:get_env(modules_loaded_file) of + case get_env(modules_loaded_file) of undefined -> ignore; File -> unload_modules(File) @@ -79,7 +81,7 @@ unload(ModuleName) -> -spec(reload(module()) -> ok | ignore | {error, any()}). reload(emqx_mod_acl_internal) -> - Modules = emqx:get_env(modules, []), + Modules = get_env(modules, []), Env = proplists:get_value(emqx_mod_acl_internal, Modules, undefined), case emqx_mod_acl_internal:reload(Env) of ok -> @@ -96,7 +98,7 @@ find_module(ModuleName) -> ets:lookup(?MODULE, ModuleName). filter_module(ModuleNames) -> - filter_module(ModuleNames, emqx:get_env(modules, [])). + filter_module(ModuleNames, get_env(modules, [])). filter_module([], Acc) -> Acc; filter_module([{ModuleName, true} | ModuleNames], Acc) -> @@ -123,7 +125,7 @@ load_module(ModuleName) -> load_module({ModuleName, true}). load_module(ModuleName, Persistent) -> - Modules = emqx:get_env(modules, []), + Modules = get_env(modules, []), Env = proplists:get_value(ModuleName, Modules, undefined), case ModuleName:load(Env) of ok -> @@ -150,7 +152,7 @@ unload_module(ModuleName) -> unload_module({ModuleName, true}). unload_module(ModuleName, Persistent) -> - Modules = emqx:get_env(modules, []), + Modules = get_env(modules, []), Env = proplists:get_value(ModuleName, Modules, undefined), case ModuleName:unload(Env) of ok -> @@ -162,7 +164,7 @@ unload_module(ModuleName, Persistent) -> end. write_loaded(true) -> - FilePath = emqx:get_env(modules_loaded_file), + FilePath = get_env(modules_loaded_file), case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- list()]) of ok -> ok; {error, Error} -> @@ -170,3 +172,7 @@ write_loaded(true) -> ok end; write_loaded(false) -> ok. + +get_env(Key) -> get_env(Key, undefined). + +get_env(Key, Default) -> application:get_env(?APP, Key, Default). diff --git a/lib-opensource/emqx_modules/src/emqx_modules_app.erl b/lib-opensource/emqx_modules/src/emqx_modules_app.erl index 832a39c8e..9eeb0bd33 100644 --- a/lib-opensource/emqx_modules/src/emqx_modules_app.erl +++ b/lib-opensource/emqx_modules/src/emqx_modules_app.erl @@ -24,13 +24,28 @@ -export([stop/1]). +-define(APP, emqx_modules). + start(_Type, _Args) -> % the configs for emqx_modules is so far still in emqx application % Ensure it's loaded application:load(emqx), + ok = load_app_env(), {ok, Pid} = emqx_mod_sup:start_link(), ok = emqx_modules:load(), {ok, Pid}. stop(_State) -> emqx_modules:unload(). + +load_app_env() -> + Schema = filename:join([code:priv_dir(?APP), "emqx_modules.schema"]), + Conf1 = filename:join([code:lib_dir(?APP), "etc", "emqx_modules.conf"]), + Conf2 = filename:join([emqx:get_env(plugins_etc_dir), "emqx_modules.conf"]), + [ConfFile | _] = lists:filter(fun filelib:is_regular/1, [Conf1, Conf2]), + Conf = cuttlefish_conf:file(ConfFile), + AppEnv = cuttlefish_generator:map(cuttlefish_schema:files([Schema]), Conf), + lists:foreach(fun({AppName, Envs}) -> + [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] + end, AppEnv). + diff --git a/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl b/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl index b0ff65758..ffbf5e32f 100644 --- a/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl @@ -24,20 +24,20 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules], fun set_sepecial_cfg/1), + emqx_ct_helpers:start_apps([emqx_modules]), + File = emqx_ct_helpers:deps_path(emqx_modules, "test/emqx_modules_SUITE_data/loaded_modules"), + application:set_env(emqx_modules, modules_loaded_file, File), + ok = emqx_modules:unload(), + ok = emqx_modules:load(), Config. -set_sepecial_cfg(_) -> - application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), - ok. - end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_modules]). t_load(_) -> ?assertEqual(ok, emqx_modules:unload()), ?assertEqual(ok, emqx_modules:load()), - ?assertEqual({error, not_found}, emqx_modules:load(not_existed_module)), + ?assertEqual({error, not_found}, emqx_modules:load(foo)), ?assertEqual({error, not_started}, emqx_modules:unload(emqx_mod_rewrite)), ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)), ?assertEqual(ok, emqx_modules:reload(emqx_mod_acl_internal)). diff --git a/test/emqx_SUITE_data/loaded_modules b/lib-opensource/emqx_modules/test/emqx_modules_SUITE_data/loaded_modules similarity index 100% rename from test/emqx_SUITE_data/loaded_modules rename to lib-opensource/emqx_modules/test/emqx_modules_SUITE_data/loaded_modules diff --git a/priv/emqx.schema b/priv/emqx.schema index bdf8a053f..ea7bde6a5 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2020,93 +2020,6 @@ end}. ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) end}. -%%-------------------------------------------------------------------- -%% Modules -%%-------------------------------------------------------------------- - -{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [ - {datatype, string} -]}. - -{mapping, "module.presence.qos", "emqx.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.topic", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.subscription.$id.qos", "emqx.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.nl", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rap", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rh", "emqx.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.rewrite.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [ - {datatype, string} -]}. - -{translation, "emqx.modules", fun(Conf, _, Conf1) -> - Subscriptions = fun() -> - List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), - TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], - [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), - nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), - rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), - rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) - }} || {N, T} <- TopicList] - end, - Rewrites = fun() -> - Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), - PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), - SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), - TotalRules = lists:append( - [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, - [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules - ), - lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> - [Topic, Re, Dest] = string:tokens(Rule, " "), - {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} - end, TotalRules) - end, - lists:append([ - [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], - [{emqx_mod_subscription, Subscriptions()}], - [{emqx_mod_rewrite, Rewrites()}], - [{emqx_mod_topic_metrics, []}], - [{emqx_mod_delayed, []}], - [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] - ]) -end}. - %%------------------------------------------------------------------- %% Plugins %%------------------------------------------------------------------- diff --git a/test/emqx_acl_cache_SUITE.erl b/test/emqx_acl_cache_SUITE.erl index 6d314598c..2adbed03c 100644 --- a/test/emqx_acl_cache_SUITE.erl +++ b/test/emqx_acl_cache_SUITE.erl @@ -56,7 +56,7 @@ t_clean_acl_cache(_) -> emqtt:stop(Client). % optimize?? -t_reload_aclfile_and_cleanall(Config) -> +t_reload_aclfile_and_cleanall(_Config) -> RasieMsg = fun() -> Self = self(), #{puback => fun(Msg) -> Self ! {puback, Msg} end, disconnected => fun(_) -> ok end, @@ -78,27 +78,6 @@ t_reload_aclfile_and_cleanall(Config) -> %% Check acl cache list [ClientPid] = emqx_cm:lookup_channels(<<"emqx_c">>), ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0), - - %% Update acl file and reload mod_acl_internal - Path = filename:join([testdir(proplists:get_value(data_dir, Config)), "acl2.conf"]), - ok = file:write_file(Path, <<"{deny, all}.">>), - OldPath = emqx:get_env(acl_file), - % application:set_env(emqx, acl_file, Path), - emqx_mod_acl_internal:reload([{acl_file, Path}]), - - ?assert(length(gen_server:call(ClientPid, list_acl_cache)) == 0), - {ok, PktId2} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1), - - receive - {puback, #{packet_id := PktId2, reason_code := Rc2}} -> - %% Not authorized - ?assertEqual(16#87, Rc2); - _ -> - ?assert(false) - end, - application:set_env(emqx, acl_file, OldPath), - file:delete(Path), - emqx_mod_acl_internal:reload([{acl_file, OldPath}]), emqtt:stop(Client). %% @private diff --git a/test/mqtt_protocol_v5_SUITE.erl b/test/mqtt_protocol_v5_SUITE.erl index fd798e759..7b8d2624e 100644 --- a/test/mqtt_protocol_v5_SUITE.erl +++ b/test/mqtt_protocol_v5_SUITE.erl @@ -179,10 +179,7 @@ t_batch_subscribe(_) -> {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"batch_test">>}]), {ok, _} = emqtt:connect(Client), application:set_env(emqx, enable_acl_cache, false), - TempAcl = emqx_ct_helpers:deps_path(emqx, "test/emqx_access_SUITE_data/acl_temp.conf"), - file:write_file(TempAcl, "{deny, {client, \"batch_test\"}, subscribe, [\"t1\", \"t2\", \"t3\"]}.\n"), - timer:sleep(10), - emqx_mod_acl_internal:reload([{acl_file, TempAcl}]), + application:set_env(emqx, acl_nomatch, deny), {ok, _, [?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED]} = emqtt:subscribe(Client, [{<<"t1">>, qos1}, @@ -193,7 +190,7 @@ t_batch_subscribe(_) -> ?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(Client, [<<"t1">>, <<"t2">>, <<"t3">>]), - file:delete(TempAcl), + application:set_env(emqx, acl_nomatch, allow), emqtt:disconnect(Client). t_connect_will_retain(_) -> From d74bb81ff13587b806e93037160dc67824a48caa Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 9 Feb 2021 23:29:16 +0100 Subject: [PATCH 07/12] chore(style): make elvis happy --- test/emqx_acl_cache_SUITE.erl | 3 ++- test/mqtt_protocol_v5_SUITE.erl | 48 +++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/test/emqx_acl_cache_SUITE.erl b/test/emqx_acl_cache_SUITE.erl index 2adbed03c..8c7685aa2 100644 --- a/test/emqx_acl_cache_SUITE.erl +++ b/test/emqx_acl_cache_SUITE.erl @@ -62,7 +62,8 @@ t_reload_aclfile_and_cleanall(_Config) -> disconnected => fun(_) -> ok end, publish => fun(_) -> ok end } end, - {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5}, {msg_handler, RasieMsg()}]), + {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5}, + {msg_handler, RasieMsg()}]), {ok, _} = emqtt:connect(Client), {ok, PktId} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1), diff --git a/test/mqtt_protocol_v5_SUITE.erl b/test/mqtt_protocol_v5_SUITE.erl index 7b8d2624e..f4d92fe8b 100644 --- a/test/mqtt_protocol_v5_SUITE.erl +++ b/test/mqtt_protocol_v5_SUITE.erl @@ -111,11 +111,13 @@ t_basic_test(_) -> t_connect_clean_start(_) -> process_flag(trap_exit, true), - {ok, Client1} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},{proto_ver, v5},{clean_start, true}]), + {ok, Client1} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>}, + {proto_ver, v5},{clean_start, true}]), {ok, _} = emqtt:connect(Client1), ?assertEqual(0, client_info(session_present, Client1)), %% [MQTT-3.1.2-4] ok = emqtt:pause(Client1), - {ok, Client2} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},{proto_ver, v5},{clean_start, false}]), + {ok, Client2} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>}, + {proto_ver, v5},{clean_start, false}]), {ok, _} = emqtt:connect(Client2), ?assertEqual(1, client_info(session_present, Client2)), %% [MQTT-3.1.2-5] ?assertEqual(142, receive_disconnect_reasoncode()), @@ -124,7 +126,8 @@ t_connect_clean_start(_) -> ok = emqtt:disconnect(Client2), waiting_client_process_exit(Client2), - {ok, Client3} = emqtt:start_link([{clientid, <<"new_client">>},{proto_ver, v5},{clean_start, false}]), + {ok, Client3} = emqtt:start_link([{clientid, <<"new_client">>}, + {proto_ver, v5},{clean_start, false}]), {ok, _} = emqtt:connect(Client3), ?assertEqual(0, client_info(session_present, Client3)), %% [MQTT-3.1.2-6] ok = emqtt:disconnect(Client3), @@ -145,7 +148,8 @@ t_connect_will_message(_) -> ]), {ok, _} = emqtt:connect(Client1), [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client1)), - ?assertNotEqual(undefined, maps:find(will_msg, emqx_connection:info(sys:get_state(ClientPid)))), %% [MQTT-3.1.2-7] + Info = emqx_connection:info(sys:get_state(ClientPid)), + ?assertNotEqual(undefined, maps:find(will_msg, Info)), %% [MQTT-3.1.2-7] {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), {ok, _} = emqtt:connect(Client2), @@ -258,9 +262,10 @@ t_connect_limit_timeout(_) -> [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client)), ?assertEqual(undefined, emqx_connection:info(limit_timer, sys:get_state(ClientPid))), - ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0), - ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0), - ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0), + Payload = <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, + ok = emqtt:publish(Client, Topic, Payload, 0), + ok = emqtt:publish(Client, Topic, Payload, 0), + ok = emqtt:publish(Client, Topic, Payload, 0), timer:sleep(200), ?assert(is_reference(emqx_connection:info(limit_timer, sys:get_state(ClientPid)))), @@ -520,7 +525,8 @@ t_publish_rap(_) -> {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), {ok, _} = emqtt:connect(Client1), {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{rap, true}, {qos, 2}]}]), - {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]), + {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"retained message">>, + [{qos, ?QOS_1}, {retain, true}]), [Msg1 | _] = receive_messages(1), ?assertEqual(true, maps:get(retain, Msg1)), %% [MQTT-3.3.1-12] ok = emqtt:disconnect(Client1), @@ -528,7 +534,8 @@ t_publish_rap(_) -> {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), {ok, _} = emqtt:connect(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, #{}, [{Topic, [{rap, false}, {qos, 2}]}]), - {ok, _} = emqtt:publish(Client2, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]), + {ok, _} = emqtt:publish(Client2, Topic, #{}, <<"retained message">>, + [{qos, ?QOS_1}, {retain, true}]), [Msg2 | _] = receive_messages(1), ?assertEqual(false, maps:get(retain, Msg2)), %% [MQTT-3.3.1-13] ok = emqtt:disconnect(Client2), @@ -572,8 +579,10 @@ t_publish_topic_alias(_) -> {ok, Client2} = emqtt:start_link([{proto_ver, v5}]), {ok, _} = emqtt:connect(Client2), {ok, _, [2]} = emqtt:subscribe(Client2, Topic, qos2), - ok = emqtt:publish(Client2, Topic, #{'Topic-Alias' => 233}, <<"Topic-Alias">>, [{qos, ?QOS_0}]), - ok = emqtt:publish(Client2, <<"">>, #{'Topic-Alias' => 233}, <<"Topic-Alias">>, [{qos, ?QOS_0}]), + ok = emqtt:publish(Client2, Topic, #{'Topic-Alias' => 233}, + <<"Topic-Alias">>, [{qos, ?QOS_0}]), + ok = emqtt:publish(Client2, <<"">>, #{'Topic-Alias' => 233}, + <<"Topic-Alias">>, [{qos, ?QOS_0}]), ?assertEqual(2, length(receive_messages(2))), %% [MQTT-3.3.2-12] ok = emqtt:disconnect(Client2), waiting_client_process_exit(Client2), @@ -586,7 +595,8 @@ t_publish_response_topic(_) -> {ok, Client1} = emqtt:start_link([{proto_ver, v5}]), {ok, _} = emqtt:connect(Client1), - ok = emqtt:publish(Client1, Topic, #{'Response-Topic' => nth(1, ?WILD_TOPICS)}, <<"Response-Topic">>, [{qos, ?QOS_0}]), + ok = emqtt:publish(Client1, Topic, #{'Response-Topic' => nth(1, ?WILD_TOPICS)}, + <<"Response-Topic">>, [{qos, ?QOS_0}]), ?assertEqual(130, receive_disconnect_reasoncode()), %% [MQTT-3.3.2-14] waiting_client_process_exit(Client1), @@ -617,7 +627,8 @@ t_publish_overlapping_subscriptions(_) -> {ok, _} = emqtt:connect(Client1), {ok, _, [1]} = emqtt:subscribe(Client1, Properties, nth(1, ?WILD_TOPICS), qos1), {ok, _, [0]} = emqtt:subscribe(Client1, Properties, nth(3, ?WILD_TOPICS), qos0), - {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"t_publish_overlapping_subscriptions">>, [{qos, ?QOS_2}]), + {ok, _} = emqtt:publish(Client1, Topic, #{}, + <<"t_publish_overlapping_subscriptions">>, [{qos, ?QOS_2}]), [Msg1 | _ ] = receive_messages(2), ?assert( maps:get(qos, Msg1) < 2 ), %% [MQTT-3.3.4-2] @@ -681,8 +692,9 @@ t_subscribe_actions(_) -> {ok, _} = emqtt:publish(Client1, Topic, <<"t_subscribe_actions">>, 2), [Msg1 | _ ] = receive_messages(1), ?assertEqual(1, maps:get(qos, Msg1)), %% [MQTT-3.8.4-3] [MQTT-3.8.4-8] - - {ok, _, [2,2]} = emqtt:subscribe(Client1, [{nth(1, ?TOPICS), qos2}, {nth(2, ?TOPICS), qos2}] ), %% [MQTT-3.8.4-5] [MQTT-3.8.4-6] [MQTT-3.8.4-7] + %% [MQTT-3.8.4-5] [MQTT-3.8.4-6] [MQTT-3.8.4-7] + {ok, _, [2,2]} = emqtt:subscribe(Client1, [{nth(1, ?TOPICS), qos2}, + {nth(2, ?TOPICS), qos2}] ), ok = emqtt:disconnect(Client1). %%-------------------------------------------------------------------- %% Unsubsctibe Unsuback @@ -699,7 +711,8 @@ t_unscbsctibe(_) -> {ok, _, [17]} = emqtt:unsubscribe(Client1, <<"noExistTopic">>), %% [MQTT-3.10.4-5] {ok, _, [2, 2]} = emqtt:subscribe(Client1, [{Topic1, qos2}, {Topic2, qos2}]), - {ok, _, [0, 0, 17]} = emqtt:unsubscribe(Client1, [Topic1, Topic2, <<"noExistTopic">>]), %% [[MQTT-3.10.4-6]] [MQTT-3.11.3-1] [MQTT-3.11.3-2] + %% [[MQTT-3.10.4-6]] [MQTT-3.11.3-1] [MQTT-3.11.3-2] + {ok, _, [0, 0, 17]} = emqtt:unsubscribe(Client1, [Topic1, Topic2, <<"noExistTopic">>]), ok = emqtt:disconnect(Client1). %%-------------------------------------------------------------------- @@ -745,7 +758,8 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(_) -> {ok, Pub} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"pub_client">>}]), {ok, _} = emqtt:connect(Pub), - {ok, _} = emqtt:publish(Pub, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 2), + {ok, _} = emqtt:publish(Pub, Topic, + <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 2), receive {'EXIT', _,{shutdown, for_testiong}} -> From 507759b9f12771b78af6ef3d8932a31514f6258f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 11 Feb 2021 15:19:40 +0100 Subject: [PATCH 08/12] Revert "refactor(emqx_modules): Move modules config to app dir" This reverts commit 9cae8784c06948867194fa3e27a790ff4c06f066. Because the reverted change broke compatibility with 4.2.x configuration. --- .../test/props/prop_webhook_confs.erl | 1 + etc/emqx.conf | 58 ++++++++++++ .../emqx_modules/etc/emqx_modules.conf | 58 +----------- .../emqx_modules/priv/emqx_modules.schema | 90 +------------------ .../src/emqx_mod_acl_internal.erl | 8 +- .../emqx_modules/src/emqx_modules.erl | 20 ++--- .../emqx_modules/src/emqx_modules_app.erl | 15 ---- .../emqx_modules/test/emqx_modules_SUITE.erl | 12 +-- priv/emqx.schema | 87 ++++++++++++++++++ .../emqx_SUITE_data}/loaded_modules | 0 test/emqx_acl_cache_SUITE.erl | 23 ++++- test/mqtt_protocol_v5_SUITE.erl | 8 +- 12 files changed, 190 insertions(+), 190 deletions(-) rename {lib-opensource/emqx_modules/test/emqx_modules_SUITE_data => test/emqx_SUITE_data}/loaded_modules (100%) diff --git a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl index 8a637c720..24903ddec 100644 --- a/apps/emqx_web_hook/test/props/prop_webhook_confs.erl +++ b/apps/emqx_web_hook/test/props/prop_webhook_confs.erl @@ -62,6 +62,7 @@ do_teardown(_) -> set_special_cfgs(_) -> application:set_env(emqx, plugins_loaded_file, undefined), + application:set_env(emqx, modules_loaded_file, undefined), ok. assert_confs([{"web.hook.api.url", Url}|More], Envs) -> diff --git a/etc/emqx.conf b/etc/emqx.conf index 9259b1de1..33db7abb6 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -2027,6 +2027,64 @@ listener.wss.external.allow_origin_absence = true ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 +##-------------------------------------------------------------------- +## Modules +##-------------------------------------------------------------------- +## The file to store loaded module names. +## +## Value: File +modules.loaded_file = {{ platform_data_dir }}/loaded_modules + +##-------------------------------------------------------------------- +## Presence Module + +## Sets the QoS for presence MQTT message. +## +## Value: 0 | 1 | 2 +module.presence.qos = 1 + +##-------------------------------------------------------------------- +## Subscription Module + +## Subscribe the Topics automatically when client connected. +## +## Value: String +## module.subscription.1.topic = connected/%c/%u + +## Qos of the proxy subscription. +## +## Value: 0 | 1 | 2 +## Default: 0 +## module.subscription.1.qos = 0 + +## No Local of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 +## Default: 0 +## module.subscription.1.nl = 0 + +## Retain As Published of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 +## Default: 0 +## module.subscription.1.rap = 0 + +## Retain Handling of the proxy subscription options. +## This configuration only takes effect in the MQTT V5 protocol. +## +## Value: 0 | 1 | 2 +## Default: 0 +## module.subscription.1.rh = 0 + +##-------------------------------------------------------------------- +## Rewrite Module + +## {rewrite, Topic, Re, Dest} +## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 +## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 + ##------------------------------------------------------------------- ## Plugins ##------------------------------------------------------------------- diff --git a/lib-opensource/emqx_modules/etc/emqx_modules.conf b/lib-opensource/emqx_modules/etc/emqx_modules.conf index 218d86234..1bb8bf6d7 100644 --- a/lib-opensource/emqx_modules/etc/emqx_modules.conf +++ b/lib-opensource/emqx_modules/etc/emqx_modules.conf @@ -1,57 +1 @@ -##-------------------------------------------------------------------- -## Modules -##-------------------------------------------------------------------- -## The file to store loaded module names. -## -## Value: File -modules.loaded_file = {{ platform_data_dir }}/loaded_modules - -##-------------------------------------------------------------------- -## Presence Module - -## Sets the QoS for presence MQTT message. -## -## Value: 0 | 1 | 2 -module.presence.qos = 1 - -##-------------------------------------------------------------------- -## Subscription Module - -## Subscribe the Topics automatically when client connected. -## -## Value: String -## module.subscription.1.topic = connected/%c/%u - -## Qos of the proxy subscription. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.qos = 0 - -## No Local of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.nl = 0 - -## Retain As Published of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 -## Default: 0 -## module.subscription.1.rap = 0 - -## Retain Handling of the proxy subscription options. -## This configuration only takes effect in the MQTT V5 protocol. -## -## Value: 0 | 1 | 2 -## Default: 0 -## module.subscription.1.rh = 0 - -##-------------------------------------------------------------------- -## Rewrite Module - -## {rewrite, Topic, Re, Dest} -## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 -## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +# empty diff --git a/lib-opensource/emqx_modules/priv/emqx_modules.schema b/lib-opensource/emqx_modules/priv/emqx_modules.schema index 55606754c..d7c52c644 100644 --- a/lib-opensource/emqx_modules/priv/emqx_modules.schema +++ b/lib-opensource/emqx_modules/priv/emqx_modules.schema @@ -1,89 +1 @@ -%%-------------------------------------------------------------------- -%% Modules -%%-------------------------------------------------------------------- - -{mapping, "modules.loaded_file", "emqx_modules.modules_loaded_file", [ - {datatype, string} -]}. - -{mapping, "module.presence.qos", "emqx_modules.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.topic", "emqx_modules.modules", [ - {datatype, string} -]}. - -{mapping, "module.subscription.$id.qos", "emqx_modules.modules", [ - {default, 1}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.subscription.$id.nl", "emqx_modules.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rap", "emqx_modules.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-1"]} -]}. - -{mapping, "module.subscription.$id.rh", "emqx_modules.modules", [ - {default, 0}, - {datatype, integer}, - {validators, ["range:0-2"]} -]}. - -{mapping, "module.rewrite.rule.$id", "emqx_modules.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.pub.rule.$id", "emqx_modules.modules", [ - {datatype, string} -]}. - -{mapping, "module.rewrite.sub.rule.$id", "emqx_modules.modules", [ - {datatype, string} -]}. - -{translation, "emqx_modules.modules", fun(Conf, _, Conf1) -> - Subscriptions = fun() -> - List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), - TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], - [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), - nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), - rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), - rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) - }} || {N, T} <- TopicList] - end, - Rewrites = fun() -> - Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), - PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), - SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), - TotalRules = lists:append( - [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, - [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules - ), - lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> - [Topic, Re, Dest] = string:tokens(Rule, " "), - {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} - end, TotalRules) - end, - lists:append([ - [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], - [{emqx_mod_subscription, Subscriptions()}], - [{emqx_mod_rewrite, Rewrites()}], - [{emqx_mod_topic_metrics, []}], - [{emqx_mod_delayed, []}], - %% TODO: acl_file config should be moved to emqx_modules.conf - %% when all the plubin tests stops using it in the old way. - [{emqx_mod_acl_internal, [{acl_file, {emqx, get_env, [acl_file]}}]}] - %[{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] - ]) -end}. +% empty diff --git a/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl b/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl index 2d347151d..1701de69d 100644 --- a/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl +++ b/lib-opensource/emqx_modules/src/emqx_mod_acl_internal.erl @@ -43,13 +43,7 @@ %%-------------------------------------------------------------------- load(Env) -> - %% TODO: acl_file config should be moved to emqx_modules.conf - %% when all the plubin tests stops using it in the old way. - File = case proplists:get_value(acl_file, Env) of - {emqx, get_env, _} -> emqx:get_env(acl_file); - F -> F - end, - Rules = rules_from_file(File), + Rules = rules_from_file(proplists:get_value(acl_file, Env)), emqx_hooks:add('client.check_acl', {?MODULE, check_acl, [Rules]}, -1). unload(_Env) -> diff --git a/lib-opensource/emqx_modules/src/emqx_modules.erl b/lib-opensource/emqx_modules/src/emqx_modules.erl index 18c6fb856..ffcc456f0 100644 --- a/lib-opensource/emqx_modules/src/emqx_modules.erl +++ b/lib-opensource/emqx_modules/src/emqx_modules.erl @@ -30,8 +30,6 @@ , load_module/2 ]). --define(APP, ?MODULE). - %% @doc List all available plugins -spec(list() -> [{atom(), boolean()}]). list() -> @@ -40,7 +38,7 @@ list() -> %% @doc Load all the extended modules. -spec(load() -> ok). load() -> - case get_env(modules_loaded_file) of + case emqx:get_env(modules_loaded_file) of undefined -> ok; File -> load_modules(File) @@ -61,7 +59,7 @@ load(ModuleName) -> %% @doc Unload all the extended modules. -spec(unload() -> ok). unload() -> - case get_env(modules_loaded_file) of + case emqx:get_env(modules_loaded_file) of undefined -> ignore; File -> unload_modules(File) @@ -81,7 +79,7 @@ unload(ModuleName) -> -spec(reload(module()) -> ok | ignore | {error, any()}). reload(emqx_mod_acl_internal) -> - Modules = get_env(modules, []), + Modules = emqx:get_env(modules, []), Env = proplists:get_value(emqx_mod_acl_internal, Modules, undefined), case emqx_mod_acl_internal:reload(Env) of ok -> @@ -98,7 +96,7 @@ find_module(ModuleName) -> ets:lookup(?MODULE, ModuleName). filter_module(ModuleNames) -> - filter_module(ModuleNames, get_env(modules, [])). + filter_module(ModuleNames, emqx:get_env(modules, [])). filter_module([], Acc) -> Acc; filter_module([{ModuleName, true} | ModuleNames], Acc) -> @@ -125,7 +123,7 @@ load_module(ModuleName) -> load_module({ModuleName, true}). load_module(ModuleName, Persistent) -> - Modules = get_env(modules, []), + Modules = emqx:get_env(modules, []), Env = proplists:get_value(ModuleName, Modules, undefined), case ModuleName:load(Env) of ok -> @@ -152,7 +150,7 @@ unload_module(ModuleName) -> unload_module({ModuleName, true}). unload_module(ModuleName, Persistent) -> - Modules = get_env(modules, []), + Modules = emqx:get_env(modules, []), Env = proplists:get_value(ModuleName, Modules, undefined), case ModuleName:unload(Env) of ok -> @@ -164,7 +162,7 @@ unload_module(ModuleName, Persistent) -> end. write_loaded(true) -> - FilePath = get_env(modules_loaded_file), + FilePath = emqx:get_env(modules_loaded_file), case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- list()]) of ok -> ok; {error, Error} -> @@ -172,7 +170,3 @@ write_loaded(true) -> ok end; write_loaded(false) -> ok. - -get_env(Key) -> get_env(Key, undefined). - -get_env(Key, Default) -> application:get_env(?APP, Key, Default). diff --git a/lib-opensource/emqx_modules/src/emqx_modules_app.erl b/lib-opensource/emqx_modules/src/emqx_modules_app.erl index 9eeb0bd33..832a39c8e 100644 --- a/lib-opensource/emqx_modules/src/emqx_modules_app.erl +++ b/lib-opensource/emqx_modules/src/emqx_modules_app.erl @@ -24,28 +24,13 @@ -export([stop/1]). --define(APP, emqx_modules). - start(_Type, _Args) -> % the configs for emqx_modules is so far still in emqx application % Ensure it's loaded application:load(emqx), - ok = load_app_env(), {ok, Pid} = emqx_mod_sup:start_link(), ok = emqx_modules:load(), {ok, Pid}. stop(_State) -> emqx_modules:unload(). - -load_app_env() -> - Schema = filename:join([code:priv_dir(?APP), "emqx_modules.schema"]), - Conf1 = filename:join([code:lib_dir(?APP), "etc", "emqx_modules.conf"]), - Conf2 = filename:join([emqx:get_env(plugins_etc_dir), "emqx_modules.conf"]), - [ConfFile | _] = lists:filter(fun filelib:is_regular/1, [Conf1, Conf2]), - Conf = cuttlefish_conf:file(ConfFile), - AppEnv = cuttlefish_generator:map(cuttlefish_schema:files([Schema]), Conf), - lists:foreach(fun({AppName, Envs}) -> - [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] - end, AppEnv). - diff --git a/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl b/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl index ffbf5e32f..b0ff65758 100644 --- a/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl +++ b/lib-opensource/emqx_modules/test/emqx_modules_SUITE.erl @@ -24,20 +24,20 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_modules]), - File = emqx_ct_helpers:deps_path(emqx_modules, "test/emqx_modules_SUITE_data/loaded_modules"), - application:set_env(emqx_modules, modules_loaded_file, File), - ok = emqx_modules:unload(), - ok = emqx_modules:load(), + emqx_ct_helpers:start_apps([emqx_modules], fun set_sepecial_cfg/1), Config. +set_sepecial_cfg(_) -> + application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")), + ok. + end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_modules]). t_load(_) -> ?assertEqual(ok, emqx_modules:unload()), ?assertEqual(ok, emqx_modules:load()), - ?assertEqual({error, not_found}, emqx_modules:load(foo)), + ?assertEqual({error, not_found}, emqx_modules:load(not_existed_module)), ?assertEqual({error, not_started}, emqx_modules:unload(emqx_mod_rewrite)), ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)), ?assertEqual(ok, emqx_modules:reload(emqx_mod_acl_internal)). diff --git a/priv/emqx.schema b/priv/emqx.schema index ea7bde6a5..bdf8a053f 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -2020,6 +2020,93 @@ end}. ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) end}. +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [ + {datatype, string} +]}. + +{mapping, "module.presence.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.topic", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.subscription.$id.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.nl", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rap", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rh", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.rewrite.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{translation, "emqx.modules", fun(Conf, _, Conf1) -> + Subscriptions = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), + TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], + [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), + PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), + SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), + TotalRules = lists:append( + [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, + [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules + ), + lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] + ]) +end}. + %%------------------------------------------------------------------- %% Plugins %%------------------------------------------------------------------- diff --git a/lib-opensource/emqx_modules/test/emqx_modules_SUITE_data/loaded_modules b/test/emqx_SUITE_data/loaded_modules similarity index 100% rename from lib-opensource/emqx_modules/test/emqx_modules_SUITE_data/loaded_modules rename to test/emqx_SUITE_data/loaded_modules diff --git a/test/emqx_acl_cache_SUITE.erl b/test/emqx_acl_cache_SUITE.erl index 8c7685aa2..a1ad061e6 100644 --- a/test/emqx_acl_cache_SUITE.erl +++ b/test/emqx_acl_cache_SUITE.erl @@ -56,7 +56,7 @@ t_clean_acl_cache(_) -> emqtt:stop(Client). % optimize?? -t_reload_aclfile_and_cleanall(_Config) -> +t_reload_aclfile_and_cleanall(Config) -> RasieMsg = fun() -> Self = self(), #{puback => fun(Msg) -> Self ! {puback, Msg} end, disconnected => fun(_) -> ok end, @@ -79,6 +79,27 @@ t_reload_aclfile_and_cleanall(_Config) -> %% Check acl cache list [ClientPid] = emqx_cm:lookup_channels(<<"emqx_c">>), ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0), + + %% Update acl file and reload mod_acl_internal + Path = filename:join([testdir(proplists:get_value(data_dir, Config)), "acl2.conf"]), + ok = file:write_file(Path, <<"{deny, all}.">>), + OldPath = emqx:get_env(acl_file), + % application:set_env(emqx, acl_file, Path), + emqx_mod_acl_internal:reload([{acl_file, Path}]), + + ?assert(length(gen_server:call(ClientPid, list_acl_cache)) == 0), + {ok, PktId2} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1), + + receive + {puback, #{packet_id := PktId2, reason_code := Rc2}} -> + %% Not authorized + ?assertEqual(16#87, Rc2); + _ -> + ?assert(false) + end, + application:set_env(emqx, acl_file, OldPath), + file:delete(Path), + emqx_mod_acl_internal:reload([{acl_file, OldPath}]), emqtt:stop(Client). %% @private diff --git a/test/mqtt_protocol_v5_SUITE.erl b/test/mqtt_protocol_v5_SUITE.erl index f4d92fe8b..738fec169 100644 --- a/test/mqtt_protocol_v5_SUITE.erl +++ b/test/mqtt_protocol_v5_SUITE.erl @@ -183,7 +183,11 @@ t_batch_subscribe(_) -> {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"batch_test">>}]), {ok, _} = emqtt:connect(Client), application:set_env(emqx, enable_acl_cache, false), - application:set_env(emqx, acl_nomatch, deny), + TempAcl = emqx_ct_helpers:deps_path(emqx, "test/emqx_access_SUITE_data/acl_temp.conf"), + file:write_file(TempAcl, "{deny, {client, \"batch_test\"}, subscribe, + [\"t1\", \"t2\", \"t3\"]}.\n"), + timer:sleep(10), + emqx_mod_acl_internal:reload([{acl_file, TempAcl}]), {ok, _, [?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED, ?RC_NOT_AUTHORIZED]} = emqtt:subscribe(Client, [{<<"t1">>, qos1}, @@ -194,7 +198,7 @@ t_batch_subscribe(_) -> ?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(Client, [<<"t1">>, <<"t2">>, <<"t3">>]), - application:set_env(emqx, acl_nomatch, allow), + file:delete(TempAcl), emqtt:disconnect(Client). t_connect_will_retain(_) -> From 6f736a79b787d2d1f9234d3112c2783ed9ee0f3c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 11 Feb 2021 15:35:37 +0100 Subject: [PATCH 09/12] chore(script): Allow compare base to be a commit sha1 hash --- scripts/elvis-check.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index 0e105bf99..e367dbfa9 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -23,10 +23,17 @@ if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx- chmod +x ./elvis fi -git fetch origin "$base" +if [[ "$base" =~ [0-9a-f]{8,40} ]]; then + # base is a commit sha1 + compare_base="$base" +else + remote="$(git remote -v | grep -E 'github\.com(.|/)emqx' | grep fetch | awk '{print $1}')" + git fetch "$remote" "$base" + compare_base="$remote/$base" +fi git_diff() { - git diff --name-only origin/"$base"...HEAD + git diff --name-only "$compare_base"...HEAD } bad_file_count=0 From 8f477e8411009d96f6f97f2f9a468d0b889a9715 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 10 Feb 2021 22:09:41 +0100 Subject: [PATCH 10/12] refactor(proj): Move opensource apps to lib-opensource One step closer for better code abstraction for opensorce vs enterprise So far this relocation is to make opensource to enterprise merges conflict free. --- .gitignore | 2 +- {apps => lib-opensource}/emqx_dashboard/.gitignore | 0 {apps => lib-opensource}/emqx_dashboard/README.md | 0 {apps => lib-opensource}/emqx_dashboard/etc/emqx_dashboard.conf | 0 .../emqx_dashboard/include/emqx_dashboard.hrl | 0 .../emqx_dashboard/priv/emqx_dashboard.schema | 0 {apps => lib-opensource}/emqx_dashboard/rebar.config | 0 .../emqx_dashboard/src/emqx_dashboard.app.src | 0 {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard.erl | 0 .../emqx_dashboard/src/emqx_dashboard_admin.erl | 0 .../emqx_dashboard/src/emqx_dashboard_api.erl | 0 .../emqx_dashboard/src/emqx_dashboard_app.erl | 0 .../emqx_dashboard/src/emqx_dashboard_cli.erl | 0 .../emqx_dashboard/src/emqx_dashboard_sup.erl | 0 {apps => lib-opensource}/emqx_dashboard/test/.placeholder | 0 .../emqx_dashboard/test/emqx_dashboard_SUITE.erl | 0 {apps => lib-opensource}/emqx_management/.gitignore | 0 {apps => lib-opensource}/emqx_management/README.md | 0 .../emqx_management/etc/emqx_management.conf | 0 {apps => lib-opensource}/emqx_management/include/emqx_mgmt.hrl | 0 .../emqx_management/priv/emqx_management.schema | 0 {apps => lib-opensource}/emqx_management/rebar.config | 0 .../emqx_management/src/emqx_management.app.src | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api.erl | 0 .../emqx_management/src/emqx_mgmt_api_alarms.erl | 0 .../emqx_management/src/emqx_mgmt_api_apps.erl | 0 .../emqx_management/src/emqx_mgmt_api_banned.erl | 0 .../emqx_management/src/emqx_mgmt_api_brokers.erl | 0 .../emqx_management/src/emqx_mgmt_api_clients.erl | 0 .../emqx_management/src/emqx_mgmt_api_data.erl | 0 .../emqx_management/src/emqx_mgmt_api_listeners.erl | 0 .../emqx_management/src/emqx_mgmt_api_metrics.erl | 0 .../emqx_management/src/emqx_mgmt_api_modules.erl | 0 .../emqx_management/src/emqx_mgmt_api_nodes.erl | 0 .../emqx_management/src/emqx_mgmt_api_plugins.erl | 0 .../emqx_management/src/emqx_mgmt_api_pubsub.erl | 0 .../emqx_management/src/emqx_mgmt_api_routes.erl | 0 .../emqx_management/src/emqx_mgmt_api_stats.erl | 0 .../emqx_management/src/emqx_mgmt_api_subscriptions.erl | 0 .../emqx_management/src/emqx_mgmt_api_topic_metrics.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_app.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_auth.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_cli.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_http.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_sup.erl | 0 {apps => lib-opensource}/emqx_management/src/emqx_mgmt_util.erl | 0 .../emqx_management/test/emqx_mgmt_SUITE.erl | 0 .../emqx_management/test/emqx_mgmt_api_SUITE.erl | 0 .../emqx_management/test/etc/emqx_management.conf | 0 .../emqx_management/test/etc/emqx_reloader.conf | 0 .../emqx_management/test/rfc6455_client.erl | 0 {apps => lib-opensource}/emqx_rule_engine/.gitignore | 0 {apps => lib-opensource}/emqx_rule_engine/README.md | 0 {apps => lib-opensource}/emqx_rule_engine/docs/api_examples.md | 0 {apps => lib-opensource}/emqx_rule_engine/docs/cli_examples.md | 0 {apps => lib-opensource}/emqx_rule_engine/docs/design.md | 0 .../emqx_rule_engine/etc/emqx_rule_engine.conf | 0 .../emqx_rule_engine/include/rule_actions.hrl | 0 .../emqx_rule_engine/include/rule_engine.hrl | 0 .../emqx_rule_engine/priv/emqx_rule_engine.schema | 0 {apps => lib-opensource}/emqx_rule_engine/rebar.config | 0 .../emqx_rule_engine/src/emqx_rule_actions.erl | 0 .../emqx_rule_engine/src/emqx_rule_actions_trans.erl | 0 .../emqx_rule_engine/src/emqx_rule_engine.app.src | 0 .../emqx_rule_engine/src/emqx_rule_engine.erl | 0 .../emqx_rule_engine/src/emqx_rule_engine_api.erl | 0 .../emqx_rule_engine/src/emqx_rule_engine_app.erl | 0 .../emqx_rule_engine/src/emqx_rule_engine_cli.erl | 0 .../emqx_rule_engine/src/emqx_rule_engine_sup.erl | 0 .../emqx_rule_engine/src/emqx_rule_events.erl | 0 .../emqx_rule_engine/src/emqx_rule_funcs.erl | 0 {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_id.erl | 0 .../emqx_rule_engine/src/emqx_rule_locker.erl | 0 .../emqx_rule_engine/src/emqx_rule_maps.erl | 0 .../emqx_rule_engine/src/emqx_rule_metrics.erl | 0 .../emqx_rule_engine/src/emqx_rule_registry.erl | 0 .../emqx_rule_engine/src/emqx_rule_runtime.erl | 0 .../emqx_rule_engine/src/emqx_rule_sqlparser.erl | 0 .../emqx_rule_engine/src/emqx_rule_sqltester.erl | 0 .../emqx_rule_engine/src/emqx_rule_utils.erl | 0 .../emqx_rule_engine/src/emqx_rule_validator.erl | 0 .../emqx_rule_engine/test/emqx_rule_engine_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_events_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_id_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_maps_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_registry_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_utils_SUITE.erl | 0 .../emqx_rule_engine/test/emqx_rule_validator_SUITE.erl | 0 .../emqx_rule_engine/test/prop_rule_maps.erl | 0 92 files changed, 1 insertion(+), 1 deletion(-) rename {apps => lib-opensource}/emqx_dashboard/.gitignore (100%) rename {apps => lib-opensource}/emqx_dashboard/README.md (100%) rename {apps => lib-opensource}/emqx_dashboard/etc/emqx_dashboard.conf (100%) rename {apps => lib-opensource}/emqx_dashboard/include/emqx_dashboard.hrl (100%) rename {apps => lib-opensource}/emqx_dashboard/priv/emqx_dashboard.schema (100%) rename {apps => lib-opensource}/emqx_dashboard/rebar.config (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard.app.src (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard_admin.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard_api.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard_app.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard_cli.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/src/emqx_dashboard_sup.erl (100%) rename {apps => lib-opensource}/emqx_dashboard/test/.placeholder (100%) rename {apps => lib-opensource}/emqx_dashboard/test/emqx_dashboard_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_management/.gitignore (100%) rename {apps => lib-opensource}/emqx_management/README.md (100%) rename {apps => lib-opensource}/emqx_management/etc/emqx_management.conf (100%) rename {apps => lib-opensource}/emqx_management/include/emqx_mgmt.hrl (100%) rename {apps => lib-opensource}/emqx_management/priv/emqx_management.schema (100%) rename {apps => lib-opensource}/emqx_management/rebar.config (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_management.app.src (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_alarms.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_apps.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_banned.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_brokers.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_clients.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_data.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_listeners.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_metrics.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_modules.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_nodes.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_plugins.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_pubsub.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_routes.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_stats.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_subscriptions.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_api_topic_metrics.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_app.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_auth.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_cli.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_http.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_sup.erl (100%) rename {apps => lib-opensource}/emqx_management/src/emqx_mgmt_util.erl (100%) rename {apps => lib-opensource}/emqx_management/test/emqx_mgmt_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_management/test/emqx_mgmt_api_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_management/test/etc/emqx_management.conf (100%) rename {apps => lib-opensource}/emqx_management/test/etc/emqx_reloader.conf (100%) rename {apps => lib-opensource}/emqx_management/test/rfc6455_client.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/.gitignore (100%) rename {apps => lib-opensource}/emqx_rule_engine/README.md (100%) rename {apps => lib-opensource}/emqx_rule_engine/docs/api_examples.md (100%) rename {apps => lib-opensource}/emqx_rule_engine/docs/cli_examples.md (100%) rename {apps => lib-opensource}/emqx_rule_engine/docs/design.md (100%) rename {apps => lib-opensource}/emqx_rule_engine/etc/emqx_rule_engine.conf (100%) rename {apps => lib-opensource}/emqx_rule_engine/include/rule_actions.hrl (100%) rename {apps => lib-opensource}/emqx_rule_engine/include/rule_engine.hrl (100%) rename {apps => lib-opensource}/emqx_rule_engine/priv/emqx_rule_engine.schema (100%) rename {apps => lib-opensource}/emqx_rule_engine/rebar.config (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_actions.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_actions_trans.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine.app.src (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine_api.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine_app.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine_cli.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_engine_sup.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_events.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_funcs.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_id.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_locker.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_maps.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_metrics.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_registry.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_runtime.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_sqlparser.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_sqltester.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_utils.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/src/emqx_rule_validator.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_events_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_id_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl (100%) rename {apps => lib-opensource}/emqx_rule_engine/test/prop_rule_maps.erl (100%) diff --git a/.gitignore b/.gitignore index d320582c5..51b4acf83 100644 --- a/.gitignore +++ b/.gitignore @@ -41,5 +41,5 @@ tmp/ _packages elvis emqx_dialyzer_*_plt -apps/emqx_dashboard/priv/www +*/emqx_dashboard/priv/www dist.zip diff --git a/apps/emqx_dashboard/.gitignore b/lib-opensource/emqx_dashboard/.gitignore similarity index 100% rename from apps/emqx_dashboard/.gitignore rename to lib-opensource/emqx_dashboard/.gitignore diff --git a/apps/emqx_dashboard/README.md b/lib-opensource/emqx_dashboard/README.md similarity index 100% rename from apps/emqx_dashboard/README.md rename to lib-opensource/emqx_dashboard/README.md diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/lib-opensource/emqx_dashboard/etc/emqx_dashboard.conf similarity index 100% rename from apps/emqx_dashboard/etc/emqx_dashboard.conf rename to lib-opensource/emqx_dashboard/etc/emqx_dashboard.conf diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/lib-opensource/emqx_dashboard/include/emqx_dashboard.hrl similarity index 100% rename from apps/emqx_dashboard/include/emqx_dashboard.hrl rename to lib-opensource/emqx_dashboard/include/emqx_dashboard.hrl diff --git a/apps/emqx_dashboard/priv/emqx_dashboard.schema b/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema similarity index 100% rename from apps/emqx_dashboard/priv/emqx_dashboard.schema rename to lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema diff --git a/apps/emqx_dashboard/rebar.config b/lib-opensource/emqx_dashboard/rebar.config similarity index 100% rename from apps/emqx_dashboard/rebar.config rename to lib-opensource/emqx_dashboard/rebar.config diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/lib-opensource/emqx_dashboard/src/emqx_dashboard.app.src similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard.app.src rename to lib-opensource/emqx_dashboard/src/emqx_dashboard.app.src diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard_admin.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard_admin.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard_admin.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard_api.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard_api.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard_api.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard_app.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard_app.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard_app.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_cli.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard_cli.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard_cli.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard_cli.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl b/lib-opensource/emqx_dashboard/src/emqx_dashboard_sup.erl similarity index 100% rename from apps/emqx_dashboard/src/emqx_dashboard_sup.erl rename to lib-opensource/emqx_dashboard/src/emqx_dashboard_sup.erl diff --git a/apps/emqx_dashboard/test/.placeholder b/lib-opensource/emqx_dashboard/test/.placeholder similarity index 100% rename from apps/emqx_dashboard/test/.placeholder rename to lib-opensource/emqx_dashboard/test/.placeholder diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/lib-opensource/emqx_dashboard/test/emqx_dashboard_SUITE.erl similarity index 100% rename from apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl rename to lib-opensource/emqx_dashboard/test/emqx_dashboard_SUITE.erl diff --git a/apps/emqx_management/.gitignore b/lib-opensource/emqx_management/.gitignore similarity index 100% rename from apps/emqx_management/.gitignore rename to lib-opensource/emqx_management/.gitignore diff --git a/apps/emqx_management/README.md b/lib-opensource/emqx_management/README.md similarity index 100% rename from apps/emqx_management/README.md rename to lib-opensource/emqx_management/README.md diff --git a/apps/emqx_management/etc/emqx_management.conf b/lib-opensource/emqx_management/etc/emqx_management.conf similarity index 100% rename from apps/emqx_management/etc/emqx_management.conf rename to lib-opensource/emqx_management/etc/emqx_management.conf diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/lib-opensource/emqx_management/include/emqx_mgmt.hrl similarity index 100% rename from apps/emqx_management/include/emqx_mgmt.hrl rename to lib-opensource/emqx_management/include/emqx_mgmt.hrl diff --git a/apps/emqx_management/priv/emqx_management.schema b/lib-opensource/emqx_management/priv/emqx_management.schema similarity index 100% rename from apps/emqx_management/priv/emqx_management.schema rename to lib-opensource/emqx_management/priv/emqx_management.schema diff --git a/apps/emqx_management/rebar.config b/lib-opensource/emqx_management/rebar.config similarity index 100% rename from apps/emqx_management/rebar.config rename to lib-opensource/emqx_management/rebar.config diff --git a/apps/emqx_management/src/emqx_management.app.src b/lib-opensource/emqx_management/src/emqx_management.app.src similarity index 100% rename from apps/emqx_management/src/emqx_management.app.src rename to lib-opensource/emqx_management/src/emqx_management.app.src diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/lib-opensource/emqx_management/src/emqx_mgmt.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt.erl rename to lib-opensource/emqx_management/src/emqx_mgmt.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_alarms.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_alarms.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_alarms.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_alarms.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_apps.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_apps.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_apps.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_apps.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_banned.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_banned.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_banned.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_brokers.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_brokers.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_brokers.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_brokers.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_clients.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_clients.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_clients.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_data.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_data.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_data.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_listeners.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_listeners.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_listeners.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_metrics.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_metrics.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_metrics.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_modules.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_modules.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_modules.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_modules.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_nodes.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_nodes.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_nodes.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_plugins.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_plugins.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_plugins.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_pubsub.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_pubsub.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_pubsub.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_pubsub.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_routes.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_routes.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_routes.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_routes.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_stats.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_stats.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_stats.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_subscriptions.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_subscriptions.erl diff --git a/apps/emqx_management/src/emqx_mgmt_api_topic_metrics.erl b/lib-opensource/emqx_management/src/emqx_mgmt_api_topic_metrics.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_api_topic_metrics.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_api_topic_metrics.erl diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/lib-opensource/emqx_management/src/emqx_mgmt_app.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_app.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_app.erl diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/lib-opensource/emqx_management/src/emqx_mgmt_auth.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_auth.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_auth.erl diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/lib-opensource/emqx_management/src/emqx_mgmt_cli.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_cli.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_cli.erl diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/lib-opensource/emqx_management/src/emqx_mgmt_http.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_http.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_http.erl diff --git a/apps/emqx_management/src/emqx_mgmt_sup.erl b/lib-opensource/emqx_management/src/emqx_mgmt_sup.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_sup.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_sup.erl diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/lib-opensource/emqx_management/src/emqx_mgmt_util.erl similarity index 100% rename from apps/emqx_management/src/emqx_mgmt_util.erl rename to lib-opensource/emqx_management/src/emqx_mgmt_util.erl diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/lib-opensource/emqx_management/test/emqx_mgmt_SUITE.erl similarity index 100% rename from apps/emqx_management/test/emqx_mgmt_SUITE.erl rename to lib-opensource/emqx_management/test/emqx_mgmt_SUITE.erl diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/lib-opensource/emqx_management/test/emqx_mgmt_api_SUITE.erl similarity index 100% rename from apps/emqx_management/test/emqx_mgmt_api_SUITE.erl rename to lib-opensource/emqx_management/test/emqx_mgmt_api_SUITE.erl diff --git a/apps/emqx_management/test/etc/emqx_management.conf b/lib-opensource/emqx_management/test/etc/emqx_management.conf similarity index 100% rename from apps/emqx_management/test/etc/emqx_management.conf rename to lib-opensource/emqx_management/test/etc/emqx_management.conf diff --git a/apps/emqx_management/test/etc/emqx_reloader.conf b/lib-opensource/emqx_management/test/etc/emqx_reloader.conf similarity index 100% rename from apps/emqx_management/test/etc/emqx_reloader.conf rename to lib-opensource/emqx_management/test/etc/emqx_reloader.conf diff --git a/apps/emqx_management/test/rfc6455_client.erl b/lib-opensource/emqx_management/test/rfc6455_client.erl similarity index 100% rename from apps/emqx_management/test/rfc6455_client.erl rename to lib-opensource/emqx_management/test/rfc6455_client.erl diff --git a/apps/emqx_rule_engine/.gitignore b/lib-opensource/emqx_rule_engine/.gitignore similarity index 100% rename from apps/emqx_rule_engine/.gitignore rename to lib-opensource/emqx_rule_engine/.gitignore diff --git a/apps/emqx_rule_engine/README.md b/lib-opensource/emqx_rule_engine/README.md similarity index 100% rename from apps/emqx_rule_engine/README.md rename to lib-opensource/emqx_rule_engine/README.md diff --git a/apps/emqx_rule_engine/docs/api_examples.md b/lib-opensource/emqx_rule_engine/docs/api_examples.md similarity index 100% rename from apps/emqx_rule_engine/docs/api_examples.md rename to lib-opensource/emqx_rule_engine/docs/api_examples.md diff --git a/apps/emqx_rule_engine/docs/cli_examples.md b/lib-opensource/emqx_rule_engine/docs/cli_examples.md similarity index 100% rename from apps/emqx_rule_engine/docs/cli_examples.md rename to lib-opensource/emqx_rule_engine/docs/cli_examples.md diff --git a/apps/emqx_rule_engine/docs/design.md b/lib-opensource/emqx_rule_engine/docs/design.md similarity index 100% rename from apps/emqx_rule_engine/docs/design.md rename to lib-opensource/emqx_rule_engine/docs/design.md diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/lib-opensource/emqx_rule_engine/etc/emqx_rule_engine.conf similarity index 100% rename from apps/emqx_rule_engine/etc/emqx_rule_engine.conf rename to lib-opensource/emqx_rule_engine/etc/emqx_rule_engine.conf diff --git a/apps/emqx_rule_engine/include/rule_actions.hrl b/lib-opensource/emqx_rule_engine/include/rule_actions.hrl similarity index 100% rename from apps/emqx_rule_engine/include/rule_actions.hrl rename to lib-opensource/emqx_rule_engine/include/rule_actions.hrl diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/lib-opensource/emqx_rule_engine/include/rule_engine.hrl similarity index 100% rename from apps/emqx_rule_engine/include/rule_engine.hrl rename to lib-opensource/emqx_rule_engine/include/rule_engine.hrl diff --git a/apps/emqx_rule_engine/priv/emqx_rule_engine.schema b/lib-opensource/emqx_rule_engine/priv/emqx_rule_engine.schema similarity index 100% rename from apps/emqx_rule_engine/priv/emqx_rule_engine.schema rename to lib-opensource/emqx_rule_engine/priv/emqx_rule_engine.schema diff --git a/apps/emqx_rule_engine/rebar.config b/lib-opensource/emqx_rule_engine/rebar.config similarity index 100% rename from apps/emqx_rule_engine/rebar.config rename to lib-opensource/emqx_rule_engine/rebar.config diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_actions.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_actions.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_actions.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_actions_trans.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_actions_trans.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine.app.src similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine.app.src rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine.app.src diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine_api.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine_api.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine_api.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_app.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine_app.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine_app.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine_app.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine_cli.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine_cli.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine_cli.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_engine_sup.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_engine_sup.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_engine_sup.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_events.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_events.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_events.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_funcs.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_funcs.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_funcs.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_id.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_id.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_id.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_id.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_locker.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_locker.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_locker.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_locker.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_maps.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_maps.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_maps.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_metrics.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_metrics.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_metrics.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_metrics.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_registry.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_registry.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_registry.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_runtime.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_runtime.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_runtime.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_sqlparser.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_sqlparser.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_sqlparser.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_sqltester.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_sqltester.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_sqltester.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_utils.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_utils.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_utils.erl diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/lib-opensource/emqx_rule_engine/src/emqx_rule_validator.erl similarity index 100% rename from apps/emqx_rule_engine/src/emqx_rule_validator.erl rename to lib-opensource/emqx_rule_engine/src/emqx_rule_validator.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_events_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_events_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_id_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_id_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_id_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_maps_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_metrics_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_registry_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl diff --git a/apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl b/lib-opensource/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl similarity index 100% rename from apps/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl rename to lib-opensource/emqx_rule_engine/test/emqx_rule_validator_SUITE.erl diff --git a/apps/emqx_rule_engine/test/prop_rule_maps.erl b/lib-opensource/emqx_rule_engine/test/prop_rule_maps.erl similarity index 100% rename from apps/emqx_rule_engine/test/prop_rule_maps.erl rename to lib-opensource/emqx_rule_engine/test/prop_rule_maps.erl From e7785a63eec21c18b23430176ba1c6ea5d2e0865 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 10 Feb 2021 22:24:41 +0100 Subject: [PATCH 11/12] fix(scripts): update scripts to work with new lib-opensource dir --- Makefile | 1 + elvis.config | 4 ++-- get-dashboard.sh | 9 ++++++--- rebar.config.erl | 2 +- sync-apps.sh | 6 +++--- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 76e3802bd..81b8b4fb0 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ REBAR_VERSION = 3.14.3-emqx-4 DASHBOARD_VERSION = v4.3.0 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build +export EMQX_ENTERPRISE=false export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) PROFILE ?= emqx diff --git a/elvis.config b/elvis.config index 42f67e40f..bb551e81a 100644 --- a/elvis.config +++ b/elvis.config @@ -5,7 +5,7 @@ [ {config, [ - #{dirs => ["apps/**/src", "src"], + #{dirs => ["src", "apps/**/src", "lib-opensource/**/src"], filter => "*.erl", ruleset => erl_files, rules => [ @@ -16,7 +16,7 @@ ]}} ] }, - #{dirs => ["apps/**/test", "test"], + #{dirs => ["test", "apps/**/test", "lib-opensource/**/src"], filter => "*.erl", rules => [ {elvis_text_style, line_length, #{ limit => 100 diff --git a/get-dashboard.sh b/get-dashboard.sh index 08879351d..885a68f77 100755 --- a/get-dashboard.sh +++ b/get-dashboard.sh @@ -1,6 +1,5 @@ -#!/bin/sh +#!/bin/bash -#set -euo pipefail set -eu VERSION="$1" @@ -10,7 +9,11 @@ cd -P -- "$(dirname -- "$0")" DOWNLOAD_URL='https://github.com/emqx/emqx-dashboard-frontend/releases/download' -DASHBOARD_PATH='apps/emqx_dashboard/priv' +if [ "$EMQX_ENTERPRISE" = 'true' ] || [ "$EMQX_ENTERPRISE" == '1' ]; then + DASHBOARD_PATH='lib-enterprise/emqx_dashboard/priv' +else + DASHBOARD_PATH='lib-opensource/emqx_dashboard/priv' +fi case $(uname) in *Darwin*) SED="sed -E";; diff --git a/rebar.config.erl b/rebar.config.erl index d68d46b9b..4bd8f45e4 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -279,7 +279,7 @@ provide_bcrypt_release(ReleaseType) -> %% rebar3 does not handle umberella project's cross-app parse_transform well compile_and_load_pase_transforms(Dir) -> PtFiles = - [ "apps/emqx_rule_engine/src/emqx_rule_actions_trans.erl" + [ "lib-opensource/emqx_rule_engine/src/emqx_rule_actions_trans.erl" ], CompileOpts = [verbose,report_errors,report_warnings,return_errors,debug_info], lists:foreach(fun(PtFile) -> {ok, _Mod} = compile:file(path(Dir, PtFile), CompileOpts) end, PtFiles). diff --git a/sync-apps.sh b/sync-apps.sh index 1b4b8fcc2..e5c36228f 100755 --- a/sync-apps.sh +++ b/sync-apps.sh @@ -15,17 +15,17 @@ apps=( "emqx_auth_redis" "emqx_bridge_mqtt" "emqx_coap" -"emqx_dashboard" +# "emqx_dashboard" # moved to lib-opensource "emqx_exhook" "emqx_exproto" "emqx_lua_hook" "emqx_lwm2m" -"emqx_management" +# "emqx_management" # moved to lib-opensource "emqx_prometheus" "emqx_psk_file" "emqx_recon" "emqx_retainer" -"emqx_rule_engine" +# "emqx_rule_engine" # moved to lib-opensource "emqx_sasl" "emqx_sn" "emqx_stomp" From 23b66c0fd58b40cb499dad0649359b20280a16b0 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 10 Feb 2021 22:25:16 +0100 Subject: [PATCH 12/12] chore(rebar.config.erl): wildcard only one level for include dir --- rebar.config.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config.erl b/rebar.config.erl index 4bd8f45e4..08b3e2be9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -291,8 +291,8 @@ str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). erl_opts_i() -> [{i, "apps"}] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "**", "include"]))] ++ - [{i, Dir} || Dir <- filelib:wildcard(filename:join([extra_lib_dir(), "**", "include"]))]. + [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++ + [{i, Dir} || Dir <- filelib:wildcard(filename:join([extra_lib_dir(), "*", "include"]))]. dialyzer(Config) -> {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),