diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f8f4eb9dc..bf972f920 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -127,7 +127,7 @@ jobs: matrix: profile: # no EDGE for mac - emqx - - emqx-ee + - emqx-enterprise otp: - 24.1.5-2 macos: @@ -213,7 +213,7 @@ jobs: profile: ## all editions for linux - emqx-edge - emqx - - emqx-ee + - emqx-enterprise otp: - 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions arch: @@ -240,9 +240,9 @@ jobs: - os: raspbian10 profile: emqx - os: raspbian9 - profile: emqx-ee + profile: emqx-enterprise - os: raspbian10 - profile: emqx-ee + profile: emqx-enterprise defaults: run: @@ -333,7 +333,7 @@ jobs: profile: # all editions for docker - emqx-edge - emqx - - emqx-ee + - emqx-enterprise # NOTE: for docker, only support latest otp version, not a matrix otp: - 24.1.5-2 # update to latest @@ -403,7 +403,7 @@ jobs: profile: - emqx-edge - emqx - - emqx-ee + - emqx-enterprise otp: - 24.1.5-2 @@ -463,7 +463,7 @@ jobs: - name: update repo.emqx.io if: github.event_name == 'release' run: | - if [ "${{ matrix. profile }}" = 'emqx-ee' ]; then + if [ "${{ matrix.profile }}" = 'emqx-enterprise' ]; then BOOL_FLAG_NAME="emqx_ee" else BOOL_FLAG_NAME="emqx_ce" diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 46eb7aef7..da62e296b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -22,7 +22,7 @@ jobs: profile: - emqx-edge - emqx - - emqx-ee + - emqx-enterprise otp: - 24.1.5-2 os: @@ -53,7 +53,7 @@ jobs: matrix: profile: - emqx - - emqx-ee + - emqx-enterprise otp: - 24.1.5-2 macos: diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index fbc6ae6f8..45b387c0d 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -27,7 +27,7 @@ jobs: run: | echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials git config --global credential.helper store - make emqx-ee-zip + make emqx-enterprise-zip - uses: actions/upload-artifact@v2 with: name: emqx-broker diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index f64e4e21f..45f8d8965 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -38,9 +38,9 @@ jobs: fail-fast: false matrix: profile: - - emqx-edge - emqx - - emqx-ee + - emqx-edge + - emqx-enterprise cluster_db_backend: - mnesia - rlog @@ -88,7 +88,7 @@ jobs: matrix: profile: - emqx - # - emqx-ee # TODO test enterprise + # - emqx-enterprise # TODO test enterprise steps: - uses: actions/download-artifact@v2 diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 6a73f846b..18d995d44 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -17,7 +17,7 @@ jobs: matrix: profile: - emqx - - emqx-ee + - emqx-enterprise otp_vsn: - 24.1.5-2 @@ -68,7 +68,7 @@ jobs: if [ $PROFILE = "emqx" ];then broker="emqx-ce" else - broker="emqx-ee" + broker="emqx-enterprise" fi echo "BROKER=$broker" >> $GITHUB_ENV diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8d2da0e21..d1f8bf577 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -23,12 +23,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: set git credentials - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi - name: xref run: make xref - name: dialyzer @@ -45,12 +39,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: set git credentials - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi - name: proper run: make proper diff --git a/Makefile b/Makefile index bb7024e88..e381db3ed 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ ifeq ($(OS),Windows_NT) endif PROFILE ?= emqx -REL_PROFILES := emqx emqx-edge emqx-ee -PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-ee-pkg +REL_PROFILES := emqx emqx-edge emqx-enterprise +PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-enterprise-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default CT_NODE_NAME ?= 'test@127.0.0.1' @@ -182,7 +182,7 @@ ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) ## emqx-docker-testing -## emqx-ee-docker-testing +## emqx-enterprise-docker-testing ## is to directly copy a unzipped zip-package to a ## base image such as ubuntu20.04. Mostly for testing .PHONY: $(REL_PROFILES:%=%-docker-testing) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 486900a60..0b877cf0b 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -577,8 +577,9 @@ mqtt { ## @doc mqtt.max_topic_levels ## ValueType: Integer ## Range: [1, 65535] - ## Default: 65535 - max_topic_levels = 65535 + ## Default: 128 + ## Depth so big may lead to subscribing performance issues + max_topic_levels = 128 ## Maximum QoS allowed. ## diff --git a/apps/emqx/src/emqx_persistent_session.erl b/apps/emqx/src/emqx_persistent_session.erl index 13c74b62e..c1ed114a1 100644 --- a/apps/emqx/src/emqx_persistent_session.erl +++ b/apps/emqx/src/emqx_persistent_session.erl @@ -18,6 +18,7 @@ -export([ is_store_enabled/0 , init_db_backend/0 + , storage_type/0 ]). -export([ discard/2 @@ -82,9 +83,17 @@ init_db_backend() -> case is_store_enabled() of true -> - ok = emqx_trie:create_session_trie(), - emqx_persistent_session_mnesia_backend:create_tables(), - persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_backend), + StorageType = storage_type(), + ok = emqx_trie:create_session_trie(StorageType), + ok = emqx_session_router:create_router_tab(StorageType), + case StorageType of + disc -> + emqx_persistent_session_mnesia_disc_backend:create_tables(), + persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_disc_backend); + ram -> + emqx_persistent_session_mnesia_ram_backend:create_tables(), + persistent_term:put(?db_backend_key, emqx_persistent_session_mnesia_ram_backend) + end, ok; false -> persistent_term:put(?db_backend_key, emqx_persistent_session_dummy_backend), @@ -94,6 +103,9 @@ init_db_backend() -> is_store_enabled() -> emqx_config:get(?is_enabled_key). +storage_type() -> + emqx_config:get(?storage_type_key). + %%-------------------------------------------------------------------- %% Session message ADT API %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_persistent_session.hrl b/apps/emqx/src/emqx_persistent_session.hrl index 4cb51160a..e973a8f2b 100644 --- a/apps/emqx/src/emqx_persistent_session.hrl +++ b/apps/emqx/src/emqx_persistent_session.hrl @@ -14,9 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(SESSION_STORE, emqx_session_store). --define(SESS_MSG_TAB, emqx_session_msg). --define(MSG_TAB, emqx_persistent_msg). +-define(SESSION_STORE_DISC, emqx_session_store_disc). +-define(SESSION_STORE_RAM, emqx_session_store_ram). + +-define(SESS_MSG_TAB_DISC, emqx_session_msg_disc). +-define(SESS_MSG_TAB_RAM, emqx_session_msg_ram). + +-define(MSG_TAB_DISC, emqx_persistent_msg_disc). +-define(MSG_TAB_RAM, emqx_persistent_msg_ram). -record(session_store, { client_id :: binary() , expiry_interval :: non_neg_integer() @@ -28,6 +33,7 @@ -define(db_backend_key, [persistent_session_store, db_backend]). -define(is_enabled_key, [persistent_session_store, enabled]). +-define(storage_type_key, [persistent_session_store, storage_type]). -define(msg_retain, [persistent_session_store, max_retain_undelivered]). -define(db_backend, (persistent_term:get(?db_backend_key))). diff --git a/apps/emqx/src/emqx_persistent_session_mnesia_backend.erl b/apps/emqx/src/emqx_persistent_session_mnesia_disc_backend.erl similarity index 79% rename from apps/emqx/src/emqx_persistent_session_mnesia_backend.erl rename to apps/emqx/src/emqx_persistent_session_mnesia_disc_backend.erl index 512984845..d10649308 100644 --- a/apps/emqx/src/emqx_persistent_session_mnesia_backend.erl +++ b/apps/emqx/src/emqx_persistent_session_mnesia_disc_backend.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_persistent_session_mnesia_backend). +-module(emqx_persistent_session_mnesia_disc_backend). -include("emqx.hrl"). -include("emqx_persistent_session.hrl"). @@ -36,7 +36,7 @@ ]). create_tables() -> - ok = mria:create_table(?SESSION_STORE, [ + ok = mria:create_table(?SESSION_STORE_DISC, [ {type, set}, {rlog_shard, ?PERSISTENT_SESSION_SHARD}, {storage, disc_copies}, @@ -44,7 +44,7 @@ create_tables() -> {attributes, record_info(fields, session_store)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}]), - ok = mria:create_table(?SESS_MSG_TAB, [ + ok = mria:create_table(?SESS_MSG_TAB_DISC, [ {type, ordered_set}, {rlog_shard, ?PERSISTENT_SESSION_SHARD}, {storage, disc_copies}, @@ -53,7 +53,7 @@ create_tables() -> {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]), - ok = mria:create_table(?MSG_TAB, [ + ok = mria:create_table(?MSG_TAB_DISC, [ {type, ordered_set}, {rlog_shard, ?PERSISTENT_SESSION_SHARD}, {storage, disc_copies}, @@ -63,43 +63,43 @@ create_tables() -> {write_concurrency, true}]}]}]). first_session_message() -> - mnesia:dirty_first(?SESS_MSG_TAB). + mnesia:dirty_first(?SESS_MSG_TAB_DISC). next_session_message(Key) -> - mnesia:dirty_next(?SESS_MSG_TAB, Key). + mnesia:dirty_next(?SESS_MSG_TAB_DISC, Key). first_message_id() -> - mnesia:dirty_first(?MSG_TAB). + mnesia:dirty_first(?MSG_TAB_DISC). next_message_id(Key) -> - mnesia:dirty_next(?MSG_TAB, Key). + mnesia:dirty_next(?MSG_TAB_DISC, Key). delete_message(Key) -> - mria:dirty_delete(?MSG_TAB, Key). + mria:dirty_delete(?MSG_TAB_DISC, Key). delete_session_message(Key) -> - mria:dirty_delete(?SESS_MSG_TAB, Key). + mria:dirty_delete(?SESS_MSG_TAB_DISC, Key). put_session_store(SS) -> - mria:dirty_write(?SESSION_STORE, SS). + mria:dirty_write(?SESSION_STORE_DISC, SS). delete_session_store(ClientID) -> - mria:dirty_delete(?SESSION_STORE, ClientID). + mria:dirty_delete(?SESSION_STORE_DISC, ClientID). lookup_session_store(ClientID) -> - case mnesia:dirty_read(?SESSION_STORE, ClientID) of + case mnesia:dirty_read(?SESSION_STORE_DISC, ClientID) of [] -> none; [SS] -> {value, SS} end. put_session_message(SessMsg) -> - mria:dirty_write(?SESS_MSG_TAB, SessMsg). + mria:dirty_write(?SESS_MSG_TAB_DISC, SessMsg). put_message(Msg) -> - mria:dirty_write(?MSG_TAB, Msg). + mria:dirty_write(?MSG_TAB_DISC, Msg). get_message(MsgId) -> - case mnesia:read(?MSG_TAB, MsgId) of + case mnesia:read(?MSG_TAB_DISC, MsgId) of [] -> error({msg_not_found, MsgId}); [Msg] -> Msg end. diff --git a/apps/emqx/src/emqx_persistent_session_mnesia_ram_backend.erl b/apps/emqx/src/emqx_persistent_session_mnesia_ram_backend.erl new file mode 100644 index 000000000..f13d92a51 --- /dev/null +++ b/apps/emqx/src/emqx_persistent_session_mnesia_ram_backend.erl @@ -0,0 +1,110 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_persistent_session_mnesia_ram_backend). + +-include("emqx.hrl"). +-include("emqx_persistent_session.hrl"). + +-export([ create_tables/0 + , first_message_id/0 + , next_message_id/1 + , delete_message/1 + , first_session_message/0 + , next_session_message/1 + , delete_session_message/1 + , put_session_store/1 + , delete_session_store/1 + , lookup_session_store/1 + , put_session_message/1 + , put_message/1 + , get_message/1 + , ro_transaction/1 + ]). + +create_tables() -> + ok = mria:create_table(?SESSION_STORE_RAM, [ + {type, set}, + {rlog_shard, ?PERSISTENT_SESSION_SHARD}, + {storage, ram_copies}, + {record_name, session_store}, + {attributes, record_info(fields, session_store)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]), + + ok = mria:create_table(?SESS_MSG_TAB_RAM, [ + {type, ordered_set}, + {rlog_shard, ?PERSISTENT_SESSION_SHARD}, + {storage, ram_copies}, + {record_name, session_msg}, + {attributes, record_info(fields, session_msg)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]), + + ok = mria:create_table(?MSG_TAB_RAM, [ + {type, ordered_set}, + {rlog_shard, ?PERSISTENT_SESSION_SHARD}, + {storage, ram_copies}, + {record_name, message}, + {attributes, record_info(fields, message)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]). + +first_session_message() -> + mnesia:dirty_first(?SESS_MSG_TAB_RAM). + +next_session_message(Key) -> + mnesia:dirty_next(?SESS_MSG_TAB_RAM, Key). + +first_message_id() -> + mnesia:dirty_first(?MSG_TAB_RAM). + +next_message_id(Key) -> + mnesia:dirty_next(?MSG_TAB_RAM, Key). + +delete_message(Key) -> + mria:dirty_delete(?MSG_TAB_RAM, Key). + +delete_session_message(Key) -> + mria:dirty_delete(?SESS_MSG_TAB_RAM, Key). + +put_session_store(SS) -> + mria:dirty_write(?SESSION_STORE_RAM, SS). + +delete_session_store(ClientID) -> + mria:dirty_delete(?SESSION_STORE_RAM, ClientID). + +lookup_session_store(ClientID) -> + case mnesia:dirty_read(?SESSION_STORE_RAM, ClientID) of + [] -> none; + [SS] -> {value, SS} + end. + +put_session_message(SessMsg) -> + mria:dirty_write(?SESS_MSG_TAB_RAM, SessMsg). + +put_message(Msg) -> + mria:dirty_write(?MSG_TAB_RAM, Msg). + +get_message(MsgId) -> + case mnesia:read(?MSG_TAB_RAM, MsgId) of + [] -> error({msg_not_found, MsgId}); + [Msg] -> Msg + end. + +ro_transaction(Fun) -> + {atomic, Res} = mria:ro_transaction(?PERSISTENT_SESSION_SHARD, Fun), + Res. + diff --git a/apps/emqx/src/emqx_router_utils.erl b/apps/emqx/src/emqx_router_utils.erl index c47f2e37b..3c7047306 100644 --- a/apps/emqx/src/emqx_router_utils.erl +++ b/apps/emqx/src/emqx_router_utils.erl @@ -20,8 +20,10 @@ -export([ delete_direct_route/2 , delete_trie_route/2 + , delete_session_trie_route/2 , insert_direct_route/2 , insert_trie_route/2 + , insert_session_trie_route/2 , maybe_trans/3 ]). @@ -30,8 +32,14 @@ insert_direct_route(Tab, Route) -> insert_trie_route(RouteTab, Route = #route{topic = Topic}) -> case mnesia:wread({RouteTab, Topic}) of - [] when RouteTab =:= emqx_route -> emqx_trie:insert(Topic); - [] when RouteTab =:= emqx_session_route -> emqx_trie:insert_session(Topic); + [] -> emqx_trie:insert(Topic); + _ -> ok + end, + mnesia:write(RouteTab, Route, sticky_write). + +insert_session_trie_route(RouteTab, Route = #route{topic = Topic}) -> + case mnesia:wread({RouteTab, Topic}) of + [] -> emqx_trie:insert_session(Topic); _ -> ok end, mnesia:write(RouteTab, Route, sticky_write). @@ -39,14 +47,20 @@ insert_trie_route(RouteTab, Route = #route{topic = Topic}) -> delete_direct_route(RouteTab, Route) -> mria:dirty_delete_object(RouteTab, Route). -delete_trie_route(RouteTab, Route = #route{topic = Topic}) -> +delete_trie_route(RouteTab, Route) -> + delete_trie_route(RouteTab, Route, normal). + +delete_session_trie_route(RouteTab, Route) -> + delete_trie_route(RouteTab, Route, session). + +delete_trie_route(RouteTab, Route = #route{topic = Topic}, Type) -> case mnesia:wread({RouteTab, Topic}) of [R] when R =:= Route -> %% Remove route and trie ok = mnesia:delete_object(RouteTab, Route, sticky_write), - case RouteTab of - emqx_route -> emqx_trie:delete(Topic); - emqx_session_route -> emqx_trie:delete_session(Topic) + case Type of + normal -> emqx_trie:delete(Topic); + session -> emqx_trie:delete_session(Topic) end; [_|_] -> %% Remove route only diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 44662d9f7..e470ff478 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -161,19 +161,50 @@ roots(low) -> fields("persistent_session_store") -> [ {"enabled", sc(boolean(), - #{ default => "false" + #{ default => false + , description => """ +Use the database to store information about persistent sessions. +This makes it possible to migrate a client connection to another +cluster node if a node is stopped. +""" + })}, + {"storage_type", + sc(hoconsc:union([ram, disc]), + #{ default => disc + , description => """ +Store information about persistent sessions on disc or in ram. +If ram is chosen, all information about persistent sessions remains +as long as at least one node in a cluster is alive to keep the information. +If disc is chosen, the information is persisted on disc and will survive +cluster restart, at the price of more disc usage and less throughput. +""" })}, {"max_retain_undelivered", sc(duration(), #{ default => "1h" + , description => """ +The time messages that was not delivered to a persistent session +is stored before being garbage collected if the node the previous +session was handled on restarts of is stopped. +""" })}, {"message_gc_interval", sc(duration(), #{ default => "1h" + , description => """ +The starting interval for garbage collection of undelivered messages to +a persistent session. This affects how often the \"max_retain_undelivered\" +is checked for removal. +""" })}, {"session_message_gc_interval", sc(duration(), #{ default => "1m" + , description => """ +The starting interval for garbage collection of transient data for +persistent session messages. This does not affect the life time length +of persistent session messages. +""" })} ]; diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 99abb0c3c..aaaedcb12 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -24,12 +24,8 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -%% Mnesia bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). - -export([ create_init_tab/0 + , create_router_tab/1 , start_link/2]). %% Route APIs @@ -60,7 +56,8 @@ -type(dest() :: node() | {group(), node()}). --define(ROUTE_TAB, emqx_session_route). +-define(ROUTE_RAM_TAB, emqx_session_route_ram). +-define(ROUTE_DISC_TAB, emqx_session_route_disc). -define(SESSION_INIT_TAB, session_init_tab). @@ -68,13 +65,22 @@ %% Mnesia bootstrap %%-------------------------------------------------------------------- -mnesia(boot) -> - ok = mria:create_table(?ROUTE_TAB, [ +create_router_tab(disc) -> + ok = mria:create_table(?ROUTE_DISC_TAB, [ {type, bag}, {rlog_shard, ?ROUTE_SHARD}, {storage, disc_copies}, {record_name, route}, {attributes, record_info(fields, route)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]); +create_router_tab(ram) -> + ok = mria:create_table(?ROUTE_RAM_TAB, [ + {type, bag}, + {rlog_shard, ?ROUTE_SHARD}, + {storage, ram_copies}, + {record_name, route}, + {attributes, record_info(fields, route)}, {storage_properties, [{ets, [{read_concurrency, true}, {write_concurrency, true}]}]}]). @@ -103,11 +109,11 @@ do_add_route(Topic, SessionID) when is_binary(Topic) -> false -> case emqx_topic:wildcard(Topic) of true -> - Fun = fun emqx_router_utils:insert_trie_route/2, - emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route], + Fun = fun emqx_router_utils:insert_session_trie_route/2, + emqx_router_utils:maybe_trans(Fun, [route_tab(), Route], ?PERSISTENT_SESSION_SHARD); false -> - emqx_router_utils:insert_direct_route(?ROUTE_TAB, Route) + emqx_router_utils:insert_direct_route(route_tab(), Route) end end. @@ -136,10 +142,10 @@ do_delete_route(Topic, SessionID) -> Route = #route{topic = Topic, dest = SessionID}, case emqx_topic:wildcard(Topic) of true -> - Fun = fun emqx_router_utils:delete_trie_route/2, - emqx_router_utils:maybe_trans(Fun, [?ROUTE_TAB, Route], ?PERSISTENT_SESSION_SHARD); + Fun = fun emqx_router_utils:delete_session_trie_route/2, + emqx_router_utils:maybe_trans(Fun, [route_tab(), Route], ?PERSISTENT_SESSION_SHARD); false -> - emqx_router_utils:delete_direct_route(?ROUTE_TAB, Route) + emqx_router_utils:delete_direct_route(route_tab(), Route) end. %% @doc Print routes to a topic @@ -273,4 +279,10 @@ init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) -> %%-------------------------------------------------------------------- lookup_routes(Topic) -> - ets:lookup(?ROUTE_TAB, Topic). + ets:lookup(route_tab(), Topic). + +route_tab() -> + case emqx_persistent_session:storage_type() of + disc -> ?ROUTE_DISC_TAB; + ram -> ?ROUTE_RAM_TAB + end. diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index cdc4677f3..35c8035f0 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -187,11 +187,20 @@ suppress(Key, SuccFun, State = #{events := Events}) -> procinfo(Pid) -> [{pid, Pid} | procinfo_l(emqx_vm:get_process_gc_info(Pid))] ++ + get_proc_lib_initial_call(Pid) ++ procinfo_l(emqx_vm:get_process_info(Pid)). procinfo_l(undefined) -> []; procinfo_l(List) -> List. +get_proc_lib_initial_call(Pid) -> + case proc_lib:initial_call(Pid) of + false -> + []; + InitialCall -> + [{proc_lib_initial_call, InitialCall}] + end. + portinfo(Port) -> [{port, Port} | erlang:port_info(Port)]. diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index 1881ecb6b..2c1560237 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -20,7 +20,7 @@ %% Mnesia bootstrap -export([ mnesia/1 - , create_session_trie/0 + , create_session_trie/1 ]). -boot_mnesia({mnesia, [boot]}). @@ -48,7 +48,8 @@ -endif. -define(TRIE, emqx_trie). --define(SESSION_TRIE, emqx_session_trie). +-define(SESSION_DISC_TRIE, emqx_session_trie_disc). +-define(SESSION_RAM_TRIE, emqx_session_trie_ram). -define(PREFIX(Prefix), {Prefix, 0}). -define(TOPIC(Topic), {Topic, 1}). @@ -75,16 +76,27 @@ mnesia(boot) -> {type, ordered_set}, {storage_properties, StoreProps}]). -create_session_trie() -> +create_session_trie(disc) -> StoreProps = [{ets, [{read_concurrency, true}, {write_concurrency, true} ]}], - ok = mria:create_table(?SESSION_TRIE, + ok = mria:create_table(?SESSION_DISC_TRIE, [{rlog_shard, ?ROUTE_SHARD}, {storage, disc_copies}, {record_name, ?TRIE}, {attributes, record_info(fields, ?TRIE)}, {type, ordered_set}, + {storage_properties, StoreProps}]); +create_session_trie(ram) -> + StoreProps = [{ets, [{read_concurrency, true}, + {write_concurrency, true} + ]}], + ok = mria:create_table(?SESSION_RAM_TRIE, + [{rlog_shard, ?ROUTE_SHARD}, + {storage, ram_copies}, + {record_name, ?TRIE}, + {attributes, record_info(fields, ?TRIE)}, + {type, ordered_set}, {storage_properties, StoreProps}]). %%-------------------------------------------------------------------- @@ -98,7 +110,7 @@ insert(Topic) when is_binary(Topic) -> -spec(insert_session(emqx_topic:topic()) -> ok). insert_session(Topic) when is_binary(Topic) -> - insert(Topic, ?SESSION_TRIE). + insert(Topic, session_trie()). insert(Topic, Trie) when is_binary(Topic) -> {TopicKey, PrefixKeys} = make_keys(Topic), @@ -115,7 +127,7 @@ delete(Topic) when is_binary(Topic) -> %% @doc Delete a topic filter from the trie. -spec(delete_session(emqx_topic:topic()) -> ok). delete_session(Topic) when is_binary(Topic) -> - delete(Topic, ?SESSION_TRIE). + delete(Topic, session_trie()). delete(Topic, Trie) when is_binary(Topic) -> {TopicKey, PrefixKeys} = make_keys(Topic), @@ -131,7 +143,7 @@ match(Topic) when is_binary(Topic) -> -spec(match_session(emqx_topic:topic()) -> list(emqx_topic:topic())). match_session(Topic) when is_binary(Topic) -> - match(Topic, ?SESSION_TRIE). + match(Topic, session_trie()). match(Topic, Trie) when is_binary(Topic) -> Words = emqx_topic:words(Topic), @@ -154,7 +166,7 @@ match(Topic, Trie) when is_binary(Topic) -> empty() -> empty(?TRIE). empty_session() -> - empty(?SESSION_TRIE). + empty(session_trie()). empty(Trie) -> ets:first(Trie) =:= '$end_of_table'. @@ -164,12 +176,19 @@ lock_tables() -> -spec lock_session_tables() -> ok. lock_session_tables() -> - mnesia:write_lock_table(?SESSION_TRIE). + mnesia:write_lock_table(session_trie()). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +session_trie() -> + case emqx_persistent_session:storage_type() of + disc -> ?SESSION_DISC_TRIE; + ram -> ?SESSION_RAM_TRIE + end. + + make_keys(Topic) -> Words = emqx_topic:words(Topic), {?TOPIC(Topic), [?PREFIX(Prefix) || Prefix <- make_prefixes(Words)]}. diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 2c19df1ea..06e17513b 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -61,7 +61,7 @@ ]). -define(PROCESS_INFO_KEYS, [initial_call, - current_function, + current_stacktrace, registered_name, status, message_queue_len, diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 911341440..284cea784 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -59,7 +59,7 @@ mqtt_conf() -> max_inflight => 32,max_mqueue_len => 1000, max_packet_size => 1048576,max_qos_allowed => 2, max_subscriptions => infinity,max_topic_alias => 65535, - max_topic_levels => 65535,mqueue_default_priority => lowest, + max_topic_levels => 128,mqueue_default_priority => lowest, mqueue_priorities => disabled,mqueue_store_qos0 => true, peer_cert_as_clientid => disabled, peer_cert_as_username => disabled, @@ -200,7 +200,7 @@ t_chan_caps(_) -> #{max_clientid_len := 65535, max_qos_allowed := 2, max_topic_alias := 65535, - max_topic_levels := 65535, + max_topic_levels := 128, retain_available := true, shared_subscription := true, subscription_identifiers := true, diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index d13d3ed8b..fd9640aeb 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -50,13 +50,19 @@ groups() -> SnabbkaffeTCs = [TC || TC <- TCs, is_snabbkaffe_tc(TC)], GCTests = [TC || TC <- TCs, is_gc_tc(TC)], OtherTCs = (TCs -- SnabbkaffeTCs) -- GCTests, - [ {persistent_store_enabled, [ {group, no_kill_connection_process} - , {group, kill_connection_process} - , {group, snabbkaffe} - , {group, gc_tests} + [ {persistent_store_enabled, [ {group, ram_tables} + , {group, disc_tables} ]} , {persistent_store_disabled, [ {group, no_kill_connection_process} ]} + , { ram_tables, [], [ {group, no_kill_connection_process} + , {group, kill_connection_process} + , {group, snabbkaffe} + , {group, gc_tests}]} + , { disc_tables, [], [ {group, no_kill_connection_process} + , {group, kill_connection_process} + , {group, snabbkaffe} + , {group, gc_tests}]} , {no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]} , { kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]} , {snabbkaffe, [], [{group, tcp_snabbkaffe}, {group, quic_snabbkaffe}, {group, ws_snabbkaffe}]} @@ -76,15 +82,23 @@ is_gc_tc(TC) -> re:run(atom_to_list(TC), "^t_gc_") /= nomatch. init_per_group(persistent_store_enabled, Config) -> + [{persistent_store_enabled, true}|Config]; +init_per_group(Group, Config) when Group =:= ram_tables; Group =:= disc_tables -> %% Start Apps + Reply = case Group =:= ram_tables of + true -> ram; + false -> disc + end, emqx_common_test_helpers:boot_modules(all), meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_config, get, fun(?is_enabled_key) -> true; - (Other) -> meck:passthrough([Other]) + meck:expect(emqx_config, get, fun(?storage_type_key) -> Reply; + (?is_enabled_key) -> true; + (Other) -> meck:passthrough([Other]) end), emqx_common_test_helpers:start_apps([], fun set_special_confs/1), ?assertEqual(true, emqx_persistent_session:is_store_enabled()), - [{persistent_store_enabled, true}|Config]; + ?assertEqual(Reply, emqx_persistent_session:storage_type()), + Config; init_per_group(persistent_store_disabled, Config) -> %% Start Apps emqx_common_test_helpers:boot_modules(all), @@ -125,15 +139,20 @@ init_per_group(gc_tests, Config) -> receive stop -> ok end end), meck:new(mnesia, [non_strict, passthrough, no_history, no_link]), - meck:expect(mnesia, dirty_first, fun(?SESS_MSG_TAB) -> ets:first(SessionMsgEts); - (?MSG_TAB) -> ets:first(MsgEts); - (X) -> meck:passthrough(X) + meck:expect(mnesia, dirty_first, fun(?SESS_MSG_TAB_RAM) -> ets:first(SessionMsgEts); + (?SESS_MSG_TAB_DISC) -> ets:first(SessionMsgEts); + (?MSG_TAB_RAM) -> ets:first(MsgEts); + (?MSG_TAB_DISC) -> ets:first(MsgEts); + (X) -> meck:passthrough([X]) end), - meck:expect(mnesia, dirty_next, fun(?SESS_MSG_TAB, X) -> ets:next(SessionMsgEts, X); - (?MSG_TAB, X) -> ets:next(MsgEts, X); + meck:expect(mnesia, dirty_next, fun(?SESS_MSG_TAB_RAM, X) -> ets:next(SessionMsgEts, X); + (?SESS_MSG_TAB_DISC, X) -> ets:next(SessionMsgEts, X); + (?MSG_TAB_RAM, X) -> ets:next(MsgEts, X); + (?MSG_TAB_DISC, X) -> ets:next(MsgEts, X); (Tab, X) -> meck:passthrough([Tab, X]) end), - meck:expect(mnesia, dirty_delete, fun(?MSG_TAB, X) -> ets:delete(MsgEts, X); + meck:expect(mnesia, dirty_delete, fun(?MSG_TAB_RAM, X) -> ets:delete(MsgEts, X); + (?MSG_TAB_DISC, X) -> ets:delete(MsgEts, X); (Tab, X) -> meck:passthrough([Tab, X]) end), [{store_owner, Pid}, {session_msg_store, SessionMsgEts}, {msg_store, MsgEts} | Config]. @@ -154,7 +173,7 @@ end_per_group(gc_tests, Config) -> meck:unload(mnesia), ?config(store_owner, Config) ! stop, ok; -end_per_group(persistent_store_enabled, _Config) -> +end_per_group(Group, _Config) when Group =:= ram_tables; Group =:= disc_tables -> meck:unload(emqx_config), emqx_common_test_helpers:stop_apps([]); end_per_group(persistent_store_disabled, _Config) -> @@ -250,7 +269,7 @@ maybe_kill_connection_process(ClientId, Config) -> end. wait_for_cm_unregister(ClientId) -> - wait_for_cm_unregister(ClientId, 10). + wait_for_cm_unregister(ClientId, 100). wait_for_cm_unregister(_ClientId, 0) -> error(cm_did_not_unregister); @@ -884,12 +903,16 @@ t_snabbkaffe_buffered_messages(Config) -> %% Make the resume init phase wait until the first message is delivered. ?force_ordering( #{ ?snk_kind := ps_worker_deliver }, #{ ?snk_kind := ps_resume_end }), + Parent = self(), spawn_link(fun() -> ?block_until(#{?snk_kind := ps_marker_pendings_msgs}, infinity, 5000), - publish(Topic, Payloads2, true) + publish(Topic, Payloads2, true), + Parent ! publish_done, + ok end), {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), {ok, _} = emqtt:ConnFun(Client2), + receive publish_done -> ok after 10000 -> error(too_long_to_publish) end, Msgs = receive_messages(length(Payloads1) + length(Payloads2) + 1), ReceivedPayloads = [P || #{ payload := P } <- Msgs], ?assertEqual(lists:sort(Payloads1 ++ Payloads2), diff --git a/apps/emqx/test/emqx_sys_mon_SUITE.erl b/apps/emqx/test/emqx_sys_mon_SUITE.erl index 1997312c7..0286800de 100644 --- a/apps/emqx/test/emqx_sys_mon_SUITE.erl +++ b/apps/emqx/test/emqx_sys_mon_SUITE.erl @@ -71,23 +71,48 @@ init_per_testcase(t_sys_mon2, Config) -> (_) -> ok end), Config; +init_per_testcase(t_procinfo, Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + ok = meck:new(emqx_vm, [passthrough, no_history]), + Config; init_per_testcase(_, Config) -> emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), Config. +end_per_testcase(t_procinfo, _Config) -> + ok = meck:unload(emqx_vm), + emqx_common_test_helpers:stop_apps([]); end_per_testcase(_, _Config) -> emqx_common_test_helpers:stop_apps([]). t_procinfo(_) -> - ok = meck:new(emqx_vm, [passthrough, no_history]), - ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end), - ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> [] end), - ?assertEqual([{pid, undefined}], emqx_sys_mon:procinfo(undefined)), ok = meck:expect(emqx_vm, get_process_info, fun(_) -> [] end), ok = meck:expect(emqx_vm, get_process_gc_info, fun(_) -> undefined end), - ?assertEqual([{pid, self()}], emqx_sys_mon:procinfo(self())), - ok = meck:unload(emqx_vm). + ?assertEqual([{pid, self()}], emqx_sys_mon:procinfo(self())). + +t_procinfo_initial_call_and_stacktrace(_) -> + SomePid = proc_lib:spawn(?MODULE, some_function, [self(), arg2]), + receive + {spawned, SomePid} -> + ok + after 100 -> + error(process_not_spawned) + end, + ProcInfo = emqx_sys_mon:procinfo(SomePid), + ?assertEqual( + {?MODULE, some_function, ['Argument__1','Argument__2']}, + proplists:get_value(proc_lib_initial_call, ProcInfo)), + ?assertMatch( + [{?MODULE, some_function, 2, + [{file, _}, + {line, _}]}, + {proc_lib, init_p_do_apply, 3, + [{file, _}, + {line, _}]}], + proplists:get_value(current_stacktrace, ProcInfo)), + SomePid ! stop. t_sys_mon(_Config) -> lists:foreach( @@ -119,3 +144,10 @@ validate_sys_mon_info(PidOrPort, SysMonName, ValidateInfo, InfoOrPort) -> emqtt:stop(C). fmt(Fmt, Args) -> lists:flatten(io_lib:format(Fmt, Args)). + +some_function(Parent, _Arg2) -> + Parent ! {spawned, self()}, + receive + stop -> + ok + end. diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index 555fc357e..3086dcdd7 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). - +-import(emqx_trace_handler_SUITE, [filesync/2]). -record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). %%-------------------------------------------------------------------- @@ -237,15 +237,16 @@ t_client_event(_Config) -> emqtt:ping(Client), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"1">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"2">>, [{qos, 0}]), - ct:sleep(200), ok = emqx_trace:create([{<<"name">>, <<"test_topic">>}, {<<"type">>, <<"topic">>}, {<<"topic">>, <<"/test">>}, {<<"start_at">>, Start}]), - ct:sleep(200), + ok = filesync(Name, clientid), + ok = filesync(<<"test_topic">>, topic), {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"3">>, [{qos, 0}]), ok = emqtt:publish(Client, <<"/test">>, #{}, <<"4">>, [{qos, 0}]), ok = emqtt:disconnect(Client), - ct:sleep(200), + ok = filesync(Name, clientid), + ok = filesync(<<"test_topic">>, topic), {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)), {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)), ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]), @@ -301,7 +302,7 @@ t_download_log(_Config) -> {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]), {ok, _} = emqtt:connect(Client), [begin _ = emqtt:ping(Client) end ||_ <- lists:seq(1, 5)], - ct:sleep(100), + ok = filesync(Name, clientid), {ok, ZipFile} = emqx_trace_api:download_zip_log(#{name => Name}, []), ?assert(filelib:file_size(ZipFile) > 0), ok = emqtt:disconnect(Client), diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index 6504530f1..e010b6c5e 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -62,7 +62,9 @@ t_trace_clientid(_Config) -> emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"), {error, {handler_not_added, {file_error, ".", eisdir}}} = emqx_trace_handler:install(clientid, <<"client5">>, debug, "."), - ct:sleep(100), + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/client.log")), @@ -83,7 +85,10 @@ t_trace_clientid(_Config) -> emqtt:connect(T), emqtt:publish(T, <<"a/b/c">>, <<"hi">>), emqtt:ping(T), - ct:sleep(200), + + ok = filesync(<<"client">>, clientid), + ok = filesync(<<"client2">>, clientid), + ok = filesync(<<"client3">>, clientid), %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log". {ok, Bin} = file:read_file("tmp/client.log"), @@ -109,7 +114,8 @@ t_trace_topic(_Config) -> emqx_logger:set_log_level(debug), ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"), ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"), - ct:sleep(100), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/topic_trace_x.log")), @@ -128,7 +134,8 @@ t_trace_topic(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"x/#">>, topic), + ok = filesync(<<"y/#">>, topic), {ok, Bin} = file:read_file("tmp/topic_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -152,8 +159,8 @@ t_trace_ip_address(_Config) -> %% Start tracing ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"), ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"), - ct:sleep(100), - + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), %% Verify the tracing file exits ?assert(filelib:is_regular("tmp/ip_trace_x.log")), ?assert(filelib:is_regular("tmp/ip_trace_y.log")), @@ -173,7 +180,8 @@ t_trace_ip_address(_Config) -> emqtt:publish(T, <<"x/y/z">>, <<"hi2">>), emqtt:subscribe(T, <<"x/y/z">>), emqtt:unsubscribe(T, <<"x/y/z">>), - ct:sleep(200), + ok = filesync(<<"127.0.0.1">>, ip_address), + ok = filesync(<<"192.168.1.1">>, ip_address), {ok, Bin} = file:read_file("tmp/ip_trace_x.log"), ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])), @@ -189,3 +197,19 @@ t_trace_ip_address(_Config) -> {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>), emqtt:disconnect(T), ?assertEqual([], emqx_trace_handler:running()). + +filesync(Name, Type) -> + filesync(Name, Type, 3). + +%% sometime the handler process is not started yet. +filesync(_Name, _Type, 0) -> ok; +filesync(Name, Type, Retry) -> + try + Handler = binary_to_atom(<<"trace_", + (atom_to_binary(Type))/binary, "_", Name/binary>>), + ok = logger_disk_log_h:filesync(Handler) + catch E:R -> + ct:pal("Filesync error:~p ~p~n", [{Name, Type, Retry}, {E, R}]), + ct:sleep(100), + filesync(Name, Type, Retry - 1) + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index eb71bec29..b83d9d1af 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include("emqx_authn.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("emqx/include/logger.hrl"). -import(hoconsc, [mk/2, ref/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -590,7 +591,7 @@ listener_authenticator(delete, authenticator_move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> - move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position); + move_authenticator([authentication], ?GLOBAL, AuthenticatorID, Position); authenticator_move(post, #{bindings := #{id := _}, body := _}) -> serialize_error({missing_parameter, position}). @@ -599,7 +600,7 @@ listener_authenticator_move(post, body := #{<<"position">> := Position}}) -> with_listener(ListenerID, fun(Type, Name, ChainName) -> - move_authenitcator([listeners, Type, Name, authentication], + move_authenticator([listeners, Type, Name, authentication], ChainName, AuthenticatorID, Position) @@ -725,7 +726,9 @@ create_authenticator(ConfKeyPath, ChainName, Config) -> raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; - {error, {_, _, Reason}} -> + {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> + serialize_error(Reason); + {error, Reason} -> serialize_error(Reason) end. @@ -753,7 +756,9 @@ update_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Config) -> raw_config := AuthenticatorsConfig}} -> {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))}; - {error, {_, _, Reason}} -> + {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> + serialize_error(Reason); + {error, Reason} -> serialize_error(Reason) end. @@ -761,11 +766,13 @@ delete_authenticator(ConfKeyPath, ChainName, AuthenticatorID) -> case update_config(ConfKeyPath, {delete_authenticator, ChainName, AuthenticatorID}) of {ok, _} -> {204}; - {error, {_, _, Reason}} -> + {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> + serialize_error(Reason); + {error, Reason} -> serialize_error(Reason) end. -move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> +move_authenticator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> case parse_position(Position) of {ok, NPosition} -> case update_config( @@ -773,7 +780,9 @@ move_authenitcator(ConfKeyPath, ChainName, AuthenticatorID, Position) -> {move_authenticator, ChainName, AuthenticatorID, NPosition}) of {ok, _} -> {204}; - {error, {_, _, Reason}} -> + {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} -> + serialize_error(Reason); + {error, Reason} -> serialize_error(Reason) end; {error, Reason} -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index c50b9cef1..ec2da3237 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -160,13 +160,15 @@ authenticate(Credential, #{resource_id := ResourceId, Request = generate_request(Credential, State), case emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}) of {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; + {ok, 200, _Headers} -> {ok, #{is_superuser => false}}; {ok, 200, Headers, Body} -> ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), case safely_parse_body(ContentType, Body) of {ok, NBody} -> %% TODO: Return by user property - {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false), - user_property => maps:remove(<<"is_superuser">>, NBody)}}; + UserProperty = maps:remove(<<"is_superuser">>, NBody), + IsSuperuser = emqx_authn_utils:is_superuser(NBody), + {ok, IsSuperuser#{user_property => UserProperty}}; {error, _Reason} -> {ok, #{is_superuser => false}} end; @@ -208,9 +210,9 @@ check_url(URL) -> end. check_body(Body) -> - maps:fold(fun(_K, _V, false) -> false; - (_K, V, true) -> is_binary(V) - end, true, Body). + lists:all( + fun erlang:is_binary/1, + maps:values(Body)). default_headers() -> maps:put(<<"content-type">>, @@ -242,12 +244,9 @@ check_ssl_opts(Conf) -> end. check_headers(Conf) -> - Method = hocon_schema:get_value("config.method", Conf), + Method = to_bin(hocon_schema:get_value("config.method", Conf)), Headers = hocon_schema:get_value("config.headers", Conf), - case Method =:= get andalso maps:get(<<"content-type">>, Headers, undefined) =/= undefined of - true -> false; - false -> true - end. + Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). parse_url(URL) -> {ok, URIMap} = emqx_http_lib:uri_parse(URL), @@ -300,7 +299,7 @@ qs([], Acc) -> <<$&, Qs/binary>> = iolist_to_binary(lists:reverse(Acc)), Qs; qs([{K, V} | More], Acc) -> - qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). + qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]). serialize_body(<<"application/json">>, Body) -> emqx_json:encode(Body); @@ -327,7 +326,17 @@ may_append_body(Output, {ok, _, _, Body}) -> may_append_body(Output, {ok, _, _}) -> Output. +uri_encode(T) -> + emqx_http_lib:uri_encode(to_bin(T)). + to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B). + +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(B) when is_binary(B) -> + B; +to_bin(L) when is_list(L) -> + list_to_binary(L). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 7a24afccb..7ec7eac6d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -125,7 +125,7 @@ verify_claims(default) -> #{}; verify_claims(validator) -> [fun do_check_verify_claims/1]; verify_claims(converter) -> fun(VerifyClaims) -> - maps:to_list(VerifyClaims) + [{to_binary(K), V} || {K, V} <- maps:to_list(VerifyClaims)] end; verify_claims(_) -> undefined. @@ -201,15 +201,14 @@ create2(#{use_jwks := false, secret := Secret0, secret_base64_encoded := Base64Encoded, verify_claims := VerifyClaims}) -> - Secret = case Base64Encoded of - true -> - base64:decode(Secret0); - false -> - Secret0 - end, - JWK = jose_jwk:from_oct(Secret), - {ok, #{jwk => JWK, - verify_claims => VerifyClaims}}; + case may_decode_secret(Base64Encoded, Secret0) of + {error, Reason} -> + {error, Reason}; + Secret -> + JWK = jose_jwk:from_oct(Secret), + {ok, #{jwk => JWK, + verify_claims => VerifyClaims}} + end; create2(#{use_jwks := false, algorithm := 'public-key', @@ -234,6 +233,14 @@ create2(#{use_jwks := true, {error, Reason} end. +may_decode_secret(false, Secret) -> Secret; +may_decode_secret(true, Secret) -> + try base64:decode(Secret) + catch + error : _ -> + {error, {invalid_parameter, secret}} + end. + replace_placeholder(L, Variables) -> replace_placeholder(L, Variables, []). @@ -349,3 +356,8 @@ validate_placeholder(<<"clientid">>) -> clientid; validate_placeholder(<<"username">>) -> username. + +to_binary(A) when is_atom(A) -> + atom_to_binary(A); +to_binary(B) when is_binary(B) -> + B. diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 5b8fc24d4..c93bec582 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -134,11 +134,23 @@ test_authenticators(PathPrefix) -> uri(PathPrefix ++ ["authentication"]), ValidConfig), - InvalidConfig = ValidConfig#{method => <<"delete">>}, + {ok, 409, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + ValidConfig), + + InvalidConfig0 = ValidConfig#{method => <<"delete">>}, {ok, 400, _} = request( post, uri(PathPrefix ++ ["authentication"]), - InvalidConfig), + InvalidConfig0), + + InvalidConfig1 = ValidConfig#{method => <<"get">>, + headers => #{<<"content-type">> => <<"application/json">>}}, + {ok, 400, _} = request( + post, + uri(PathPrefix ++ ["authentication"]), + InvalidConfig1), ?assertAuthenticatorsMatch( [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], @@ -170,6 +182,13 @@ test_authenticator(PathPrefix) -> uri(PathPrefix ++ ["authentication", "password-based:http"]), InvalidConfig0), + InvalidConfig1 = ValidConfig0#{method => <<"get">>, + headers => #{<<"content-type">> => <<"application/json">>}}, + {ok, 400, _} = request( + put, + uri(PathPrefix ++ ["authentication", "password-based:http"]), + InvalidConfig1), + ValidConfig1 = ValidConfig0#{pool_size => 9}, {ok, 200, _} = request( put, diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl new file mode 100644 index 000000000..4a966fe0e --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -0,0 +1,392 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_http_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("emqx/include/emqx_placeholder.hrl"). + +-define(PATH, [authentication]). + +-define(HTTP_PORT, 33333). +-define(HTTP_PATH, "/auth"). +-define(CREDENTIALS, #{username => <<"plain">>, + password => <<"plain">>, + listener => 'tcp:default', + protocol => mqtt + }). + + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_authn]), + application:ensure_all_started(cowboy), + Config. + +end_per_suite(_) -> + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + emqx_common_test_helpers:stop_apps([emqx_authn]), + application:stop(cowboy), + ok. + +init_per_testcase(_Case, Config) -> + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + emqx_authn_http_test_server:start(?HTTP_PORT, ?HTTP_PATH), + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_authn_http_test_server:stop() . + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + AuthConfig = raw_http_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + {ok, [#{provider := emqx_authn_http}]} = emqx_authentication:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + AuthConfig = raw_http_auth_config(), + + InvalidConfigs = + [ + AuthConfig#{headers => []}, + AuthConfig#{method => delete} + ], + + lists:foreach( + fun(Config) -> + ct:pal("creating authenticator with invalid config: ~p", [Config]), + {error, _} = + try + emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}) + catch + throw:Error -> + {error, Error} + end, + {ok, []} = emqx_authentication:list_authenticators(?GLOBAL) + end, + InvalidConfigs). + +t_authenticate(_Config) -> + ok = lists:foreach( + fun(Sample) -> + ct:pal("test_user_auth sample: ~p", [Sample]), + test_user_auth(Sample) + end, + samples()). + +test_user_auth(#{handler := Handler, + config_params := SpecificConfgParams, + result := Result}) -> + AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + emqx_authn_http_test_server:set_handler(Handler), + + ?assertEqual(Result, emqx_access_control:authenticate(?CREDENTIALS)), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL). + +t_destroy(_Config) -> + AuthConfig = raw_http_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + ok = emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = cowboy_req:reply(200, Req0), + {ok, Req, State} + end), + + {ok, [#{provider := emqx_authn_http, state := State}]} + = emqx_authentication:list_authenticators(?GLOBAL), + + Credentials = maps:with([username, password], ?CREDENTIALS), + + {ok, _} = emqx_authn_http:authenticate( + Credentials, + State), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + + % Authenticator should not be usable anymore + ?assertException( + error, + _, + emqx_authn_http:authenticate( + Credentials, + State)). + +t_update(_Config) -> + CorrectConfig = raw_http_auth_config(), + IncorrectConfig = + CorrectConfig#{url => <<"http://127.0.0.1:33333/invalid">>}, + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, IncorrectConfig}), + + ok = emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = cowboy_req:reply(200, Req0), + {ok, Req, State} + end), + + {error, not_authorized} = emqx_access_control:authenticate(?CREDENTIALS), + + % We update with config with correct query, provider should update and work properly + {ok, _} = emqx:update_config( + ?PATH, + {update_authenticator, ?GLOBAL, <<"password-based:http">>, CorrectConfig}), + + {ok,_} = emqx_access_control:authenticate(?CREDENTIALS). + +t_is_superuser(_Config) -> + Config = raw_http_auth_config(), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + Checks = [ + {json, <<"0">>, false}, + {json, <<"">>, false}, + {json, null, false}, + {json, 0, false}, + + {json, <<"1">>, true}, + {json, <<"val">>, true}, + {json, 1, true}, + {json, 123, true}, + + {form, <<"0">>, false}, + {form, <<"">>, false}, + + {form, <<"1">>, true}, + {form, <<"val">>, true} + ], + + lists:foreach(fun test_is_superuser/1, Checks). + +test_is_superuser({Kind, Value, ExpectedValue}) -> + + {ContentType, Res} = case Kind of + json -> + {<<"application/json">>, + jiffy:encode(#{is_superuser => Value})}; + form -> + {<<"application/x-www-form-urlencoded">>, + iolist_to_binary([<<"is_superuser=">>, Value])} + end, + + emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => ContentType}, + Res, + Req0), + {ok, Req, State} + end), + + ?assertMatch( + {ok, #{is_superuser := ExpectedValue}}, + emqx_access_control:authenticate(?CREDENTIALS)). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +raw_http_auth_config() -> + #{ + mechanism => <<"password-based">>, + enable => <<"true">>, + + backend => <<"http">>, + method => <<"get">>, + url => <<"http://127.0.0.1:33333/auth">>, + body => #{<<"username">> => ?PH_USERNAME, <<"password">> => ?PH_PASSWORD}, + headers => #{<<"X-Test-Header">> => <<"Test Value">>} + }. + +samples() -> + [ + %% simple get request + #{handler => fun(Req0, State) -> + #{username := <<"plain">>, + password := <<"plain">> + } = cowboy_req:match_qs([username, password], Req0), + + Req = cowboy_req:reply(200, Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => false}} + }, + + %% get request with json body response + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + jiffy:encode(#{is_superuser => true}), + Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => true, user_property => #{}}} + }, + + %% get request with url-form-encoded body response + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => + <<"application/x-www-form-urlencoded">>}, + <<"is_superuser=true">>, + Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => true, user_property => #{}}} + }, + + %% get request with response of unknown encoding + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => + <<"test/plain">>}, + <<"is_superuser=true">>, + Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => false}} + }, + + %% simple post request, application/json + #{handler => fun(Req0, State) -> + {ok, RawBody, Req1} = cowboy_req:read_body(Req0), + #{<<"username">> := <<"plain">>, + <<"password">> := <<"plain">> + } = jiffy:decode(RawBody, [return_maps]), + Req = cowboy_req:reply(200, Req1), + {ok, Req, State} + end, + config_params => #{ + method => post, + headers => #{<<"content-type">> => <<"application/json">>} + }, + result => {ok,#{is_superuser => false}} + }, + + %% simple post request, application/x-www-form-urlencoded + #{handler => fun(Req0, State) -> + {ok, PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0), + #{<<"username">> := <<"plain">>, + <<"password">> := <<"plain">> + } = maps:from_list(PostVars), + Req = cowboy_req:reply(200, Req1), + {ok, Req, State} + end, + config_params => #{ + method => post, + headers => #{<<"content-type">> => + <<"application/x-www-form-urlencoded">>} + }, + result => {ok,#{is_superuser => false}} + } + + %% 204 code + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply(204, Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => false}} + }, + + %% custom headers + #{handler => fun(Req0, State) -> + <<"Test Value">> = cowboy_req:header(<<"x-test-header">>, Req0), + Req = cowboy_req:reply(200, Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {ok,#{is_superuser => false}} + }, + + %% 400 code + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply(400, Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {error,not_authorized} + }, + + %% 500 code + #{handler => fun(Req0, State) -> + Req = cowboy_req:reply(500, Req0), + {ok, Req, State} + end, + config_params => #{}, + result => {error,not_authorized} + }, + + %% Handling error + #{handler => fun(Req0, State) -> + error(woops), + {ok, Req0, State} + end, + config_params => #{}, + result => {error,not_authorized} + } + ]. + +start_apps(Apps) -> + lists:foreach(fun application:ensure_all_started/1, Apps). + +stop_apps(Apps) -> + lists:foreach(fun application:stop/1, Apps). diff --git a/apps/emqx_authn/test/emqx_authn_http_test_server.erl b/apps/emqx_authn/test/emqx_authn_http_test_server.erl new file mode 100644 index 000000000..4896a8b13 --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_http_test_server.erl @@ -0,0 +1,89 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_http_test_server). + +-behaviour(gen_server). +-behaviour(cowboy_handler). + +% cowboy_server callbacks +-export([init/2]). + +% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2 + ]). + +% API +-export([start/2, + stop/0, + set_handler/1 + ]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +start(Port, Path) -> + Dispatch = cowboy_router:compile([ + {'_', [{Path, ?MODULE, []}]} + ]), + {ok, _} = cowboy:start_clear(?MODULE, + [{port, Port}], + #{env => #{dispatch => Dispatch}} + ), + {ok, _} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []), + ok. + +stop() -> + gen_server:stop(?MODULE), + cowboy:stop_listener(?MODULE). + +set_handler(F) when is_function(F, 2) -> + gen_server:call(?MODULE, {set_handler, F}). + +%%------------------------------------------------------------------------------ +%% gen_server API +%%------------------------------------------------------------------------------ + +init([]) -> + F = fun(Req0, State) -> + Req = cowboy_req:reply( + 400, + #{<<"content-type">> => <<"text/plain">>}, + <<"">>, + Req0), + {ok, Req, State} + end, + {ok, F}. + +handle_cast(_, F) -> + {noreply, F}. + +handle_call({set_handler, F}, _From, _F) -> + {reply, ok, F}; + +handle_call(get_handler, _From, F) -> + {reply, F, F}. + +%%------------------------------------------------------------------------------ +%% cowboy_server API +%%------------------------------------------------------------------------------ + +init(Req, State) -> + Handler = gen_server:call(?MODULE, get_handler), + Handler(Req, State). diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 170449dcc..54db0a3c5 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -19,140 +19,142 @@ -compile(export_all). -compile(nowarn_export_all). -% -include_lib("common_test/include/ct.hrl"). -% -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). -% -include("emqx_authn.hrl"). +-include("emqx_authn.hrl"). -% -define(AUTH, emqx_authn). +-define(AUTHN_ID, <<"mechanism:jwt">>). all() -> emqx_common_test_helpers:all(?MODULE). -% init_per_suite(Config) -> -% emqx_common_test_helpers:start_apps([emqx_authn]), -% Config. +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_authn]), + Config. -% end_per_suite(_) -> -% emqx_common_test_helpers:stop_apps([emqx_authn]), -% ok. +end_per_suite(_) -> + emqx_common_test_helpers:stop_apps([emqx_authn]), + ok. -% t_jwt_authenticator(_) -> -% AuthenticatorName = <<"myauthenticator">>, -% Config = #{name => AuthenticatorName, -% mechanism => jwt, -% use_jwks => false, -% algorithm => 'hmac-based', -% secret => <<"abcdef">>, -% secret_base64_encoded => false, -% verify_claims => []}, -% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), +t_jwt_authenticator(_) -> + Secret = <<"abcdef">>, + Config = #{mechanism => jwt, + use_jwks => false, + algorithm => 'hmac-based', + secret => Secret, + secret_base64_encoded => false, + verify_claims => []}, + {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), -% Payload = #{<<"username">> => <<"myuser">>}, -% JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), -% ClientInfo = #{username => <<"myuser">>, -% password => JWS}, -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('hmac-based', Payload, Secret), + Credential = #{username => <<"myuser">>, + password => JWS}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)), -% Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true}, -% JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), -% ClientInfo1 = #{username => <<"myuser">>, -% password => JWS1}, -% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), + Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true}, + JWS1 = generate_jws('hmac-based', Payload1, Secret), + Credential1 = #{username => <<"myuser">>, + password => JWS1}, + ?assertEqual({ok, #{is_superuser => true}}, emqx_authn_jwt:authenticate(Credential1, State)), -% BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), -% ClientInfo2 = ClientInfo#{password => BadJWS}, -% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), + BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), + Credential2 = Credential#{password => BadJWS}, + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential2, State)), -% %% secret_base64_encoded -% Config2 = Config#{secret => base64:encode(<<"abcdef">>), -% secret_base64_encoded => true}, -% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), + %% secret_base64_encoded + Config2 = Config#{secret => base64:encode(Secret), + secret_base64_encoded => true}, + {ok, State2} = emqx_authn_jwt:update(Config2, State), + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State2)), -% Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, -% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), -% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), + %% invalid secret + BadConfig = Config#{secret => <<"emqxsecret">>, + secret_base64_encoded => true}, + {error, {invalid_parameter, secret}} = emqx_authn_jwt:create(?AUTHN_ID, BadConfig), -% %% Expiration -% Payload3 = #{ <<"username">> => <<"myuser">> -% , <<"exp">> => erlang:system_time(second) - 60}, -% JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), -% ClientInfo3 = ClientInfo#{password => JWS3}, -% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), + Config3 = Config#{verify_claims => [{<<"username">>, <<"${username}">>}]}, + {ok, State3} = emqx_authn_jwt:update(Config3, State2), + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State3)), + ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential#{username => <<"otheruser">>}, State3)), -% Payload4 = #{ <<"username">> => <<"myuser">> -% , <<"exp">> => erlang:system_time(second) + 60}, -% JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), -% ClientInfo4 = ClientInfo#{password => JWS4}, -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), + %% Expiration + Payload3 = #{ <<"username">> => <<"myuser">> + , <<"exp">> => erlang:system_time(second) - 60}, + JWS3 = generate_jws('hmac-based', Payload3, Secret), + Credential3 = Credential#{password => JWS3}, + ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential3, State3)), -% %% Issued At -% Payload5 = #{ <<"username">> => <<"myuser">> -% , <<"iat">> => erlang:system_time(second) - 60}, -% JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), -% ClientInfo5 = ClientInfo#{password => JWS5}, -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), + Payload4 = #{ <<"username">> => <<"myuser">> + , <<"exp">> => erlang:system_time(second) + 60}, + JWS4 = generate_jws('hmac-based', Payload4, Secret), + Credential4 = Credential#{password => JWS4}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential4, State3)), -% Payload6 = #{ <<"username">> => <<"myuser">> -% , <<"iat">> => erlang:system_time(second) + 60}, -% JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), -% ClientInfo6 = ClientInfo#{password => JWS6}, -% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), + %% Issued At + Payload5 = #{ <<"username">> => <<"myuser">> + , <<"iat">> => erlang:system_time(second) - 60}, + JWS5 = generate_jws('hmac-based', Payload5, Secret), + Credential5 = Credential#{password => JWS5}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential5, State3)), -% %% Not Before -% Payload7 = #{ <<"username">> => <<"myuser">> -% , <<"nbf">> => erlang:system_time(second) - 60}, -% JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), -% ClientInfo7 = ClientInfo#{password => JWS7}, -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), + Payload6 = #{ <<"username">> => <<"myuser">> + , <<"iat">> => erlang:system_time(second) + 60}, + JWS6 = generate_jws('hmac-based', Payload6, Secret), + Credential6 = Credential#{password => JWS6}, + ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential6, State3)), -% Payload8 = #{ <<"username">> => <<"myuser">> -% , <<"nbf">> => erlang:system_time(second) + 60}, -% JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), -% ClientInfo8 = ClientInfo#{password => JWS8}, -% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), + %% Not Before + Payload7 = #{ <<"username">> => <<"myuser">> + , <<"nbf">> => erlang:system_time(second) - 60}, + JWS7 = generate_jws('hmac-based', Payload7, Secret), + Credential7 = Credential6#{password => JWS7}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential7, State3)), -% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), -% ok. + Payload8 = #{ <<"username">> => <<"myuser">> + , <<"nbf">> => erlang:system_time(second) + 60}, + JWS8 = generate_jws('hmac-based', Payload8, Secret), + Credential8 = Credential#{password => JWS8}, + ?assertEqual({error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential8, State3)), -% t_jwt_authenticator2(_) -> -% Dir = code:lib_dir(emqx_authn, test), -% PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), -% PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), -% AuthenticatorName = <<"myauthenticator">>, -% Config = #{name => AuthenticatorName, -% mechanism => jwt, -% use_jwks => false, -% algorithm => 'public-key', -% certificate => PublicKey, -% verify_claims => []}, -% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), + ?assertEqual(ok, emqx_authn_jwt:destroy(State3)), + ok. -% Payload = #{<<"username">> => <<"myuser">>}, -% JWS = generate_jws('public-key', Payload, PrivateKey), -% ClientInfo = #{username => <<"myuser">>, -% password => JWS}, -% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), -% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), +t_jwt_authenticator2(_) -> + Dir = code:lib_dir(emqx_authn, test), + PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), + PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), + Config = #{mechanism => jwt, + use_jwks => false, + algorithm => 'public-key', + certificate => PublicKey, + verify_claims => []}, + {ok, State} = emqx_authn_jwt:create(?AUTHN_ID, Config), -% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), -% ok. + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('public-key', Payload, PrivateKey), + Credential = #{username => <<"myuser">>, + password => JWS}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State)), + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State)), -% generate_jws('hmac-based', Payload, Secret) -> -% JWK = jose_jwk:from_oct(Secret), -% Header = #{ <<"alg">> => <<"HS256">> -% , <<"typ">> => <<"JWT">> -% }, -% Signed = jose_jwt:sign(JWK, Header, Payload), -% {_, JWS} = jose_jws:compact(Signed), -% JWS; -% generate_jws('public-key', Payload, PrivateKey) -> -% JWK = jose_jwk:from_pem_file(PrivateKey), -% Header = #{ <<"alg">> => <<"RS256">> -% , <<"typ">> => <<"JWT">> -% }, -% Signed = jose_jwt:sign(JWK, Header, Payload), -% {_, JWS} = jose_jws:compact(Signed), -% JWS. + ?assertEqual(ok, emqx_authn_jwt:destroy(State)), + ok. + +generate_jws('hmac-based', Payload, Secret) -> + JWK = jose_jwk:from_oct(Secret), + Header = #{ <<"alg">> => <<"HS256">> + , <<"typ">> => <<"JWT">> + }, + Signed = jose_jwt:sign(JWK, Header, Payload), + {_, JWS} = jose_jws:compact(Signed), + JWS; +generate_jws('public-key', Payload, PrivateKey) -> + JWK = jose_jwk:from_pem_file(PrivateKey), + Header = #{ <<"alg">> => <<"RS256">> + , <<"typ">> => <<"JWT">> + }, + Signed = jose_jwt:sign(JWK, Header, Payload), + {_, JWS} = jose_jws:compact(Signed), + JWS. diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 371d56e30..4e0baa8fd 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -68,6 +68,8 @@ -define(CONF_KEY_PATH, [authorization, sources]). +-define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). + -define(USERNAME_RULES_EXAMPLE, #{username => user1, rules => [ #{topic => <<"test/toopic/1">>, permission => <<"allow">>, diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index 6821f15c3..a3a5e1ed9 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -38,10 +38,10 @@ description() -> parse_query(undefined) -> undefined; parse_query(Sql) -> - case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of + case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of {match, Variables} -> Params = [Var || [Var] <- Variables], - {re:replace(Sql, "'%[ucCad]'", "?", [global, {return, list}]), Params}; + {re:replace(Sql, ?RE_PLACEHOLDER, "?", [global, {return, list}]), Params}; nomatch -> {Sql, []} end. diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index d88b35b41..5bae5f674 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -38,7 +38,7 @@ description() -> parse_query(undefined) -> undefined; parse_query(Sql) -> - case re:run(Sql, "'%[ucCad]'", [global, {capture, all, list}]) of + case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of {match, Variables} -> Params = [Var || [Var] <- Variables], Vars = ["$" ++ integer_to_list(I) || I <- lists:seq(1, length(Params))], diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index c2d992cb6..11eac9c91 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -169,8 +169,11 @@ health_check(PoolName) -> case ecpool_worker:client(Worker) of {ok, Conn} -> %% we don't care if this returns something or not, we just to test the connection - Res = mongo_api:find_one(Conn, <<"foo">>, {}, #{}), - Res == undefined orelse is_map(Res); + try mongo_api:find_one(Conn, <<"foo">>, {}, #{}) of + _ -> true + catch + _Class:_Error -> false + end; _ -> false end end || {_WorkerName, Worker} <- ecpool:workers(PoolName)], diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index b6724f701..59c0f560a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -31,7 +31,8 @@ -define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])). --define(MAX_ROW_LIMIT, 100). +-define(MAX_ROW_LIMIT, 1000). +-define(DEFAULT_ROW, 100). -type(request() :: #{bindings => map(), query_string => map(), body => map()}). -type(request_meta() :: #{module => module(), path => string(), method => atom()}). @@ -80,7 +81,7 @@ fields(page) -> fields(limit) -> Desc = iolist_to_binary([<<"Results per page(max ">>, integer_to_binary(?MAX_ROW_LIMIT), <<")">>]), - Meta = #{in => query, desc => Desc, default => ?MAX_ROW_LIMIT, example => 50}, + Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50}, [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}]. -spec(schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map()). @@ -123,10 +124,10 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec {Bindings, QueryStr} = check_parameters(Request, Params, Module), NewBody = check_request_body(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)), {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}} - catch throw:Error -> - {_, [{validation_error, ValidErr}]} = Error, - #{path := Key, reason := Reason} = ValidErr, - {400, 'BAD_REQUEST', iolist_to_binary(io_lib:format("~ts : ~p", [Key, Reason]))} + catch throw:{_, ValidErrors} -> + Msg = [io_lib:format("~ts : ~p", [Key, Reason]) || + {validation_error, #{path := Key, reason := Reason}} <- ValidErrors], + {400, 'BAD_REQUEST', iolist_to_binary(string:join(Msg, ","))} end. check_and_translate(Schema, Map, Opts) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index c938788e4..3c70cfe65 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -71,8 +71,10 @@ t_public_ref(_Config) -> {emqx_dashboard_swagger, page, parameter} ], Refs), ExpectRefs = [ - #{<<"public.limit">> => #{description => <<"Results per page(max 100)">>, example => 50,in => query,name => limit, - schema => #{default => 100,example => 1,maximum => 100, minimum => 1,type => integer}}}, + #{<<"public.limit">> => #{description => <<"Results per page(max 1000)">>, + example => 50,in => query,name => limit, + schema => #{default => 100,example => 1,maximum => 1000, + minimum => 1,type => integer}}}, #{<<"public.page">> => #{description => <<"Page number of the results to fetch.">>, example => 1,in => query,name => page, schema => #{default => 1,example => 100,type => integer}}}], @@ -176,7 +178,8 @@ t_in_mix_trans(_Config) -> Expect = {ok, #{body => #{}, bindings => #{state => 720}, - query_string => #{<<"filter">> => created,<<"is_admin">> => true, <<"per_page">> => 5,<<"timeout">> => 34}}}, + query_string => #{<<"filter">> => created,<<"is_admin">> => true, + <<"per_page">> => 5,<<"timeout">> => 34}}}, ?assertEqual(Expect, trans_parameters(Path, Bindings, Query)), ok. @@ -268,7 +271,10 @@ schema("/test/in/:filter") -> parameters => [ {filter, mk(hoconsc:enum([assigned, created, mentioned, all]), - #{in => path, desc => <<"Indicates which sorts of issues to return">>, example => "all"})} + #{in => path, + desc => <<"Indicates which sorts of issues to return">>, + example => "all" + })} ], responses => #{200 => <<"ok">>} } @@ -323,9 +329,11 @@ schema("/test/in/mix/:state") -> deprecated => true, parameters => [ {filter, hoconsc:mk(hoconsc:enum([assigned, created, mentioned, all]), - #{in => query, desc => <<"Indicates which sorts of issues to return">>, example => "all"})}, + #{in => query, desc => <<"Indicates which sorts of issues to return">>, + example => "all"})}, {state, mk(emqx_schema:duration_s(), - #{in => path, required => true, example => "12m", desc => <<"Indicates the state of the issues to return.">>})}, + #{in => path, required => true, example => "12m", + desc => <<"Indicates the state of the issues to return.">>})}, {per_page, mk(range(1, 50), #{in => query, required => false, example => 10, default => 5})}, {is_admin, mk(boolean(), #{in => query})}, diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 149657ec0..ae74fc08e 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -47,9 +47,14 @@ t_object(_Config) -> #{<<"schema">> => #{required => [<<"timeout">>, <<"per_page">>], <<"properties">> =>[ - {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, - {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, - {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + {<<"per_page">>, #{description => <<"good per page desc">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, + #{enum => [infinity], type => string}]}}, + {<<"inner_ref">>, + #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], <<"type">> => object}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, good_ref}], @@ -63,14 +68,20 @@ t_nest_object(_Config) -> #{<<"schema">> => #{required => [<<"timeout">>], <<"properties">> => - [{<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + [{<<"per_page">>, #{description => <<"good per page desc">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, {<<"timeout">>, #{default => 5, <<"oneOf">> => - [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + [#{example => <<"1h">>, type => string}, + #{enum => [infinity], type => string}]}}, {<<"nest_object">>, #{<<"properties">> => [{<<"good_nest_1">>, #{example => 100, type => integer}}, - {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}],<<"type">> => object}}, - {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + {<<"good_nest_2">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + <<"type">> => object}}, + {<<"inner_ref">>, + #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], <<"type">> => object}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, good_ref}], @@ -81,7 +92,8 @@ t_local_ref(_Config) -> Spec = #{ post => #{parameters => [], requestBody => #{<<"content">> => #{<<"application/json">> => - #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}}}, + #{<<"schema">> => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, good_ref}], validate("/ref/local", Spec, Refs), @@ -91,17 +103,22 @@ t_remote_ref(_Config) -> Spec = #{ post => #{parameters => [], requestBody => #{<<"content">> => #{<<"application/json">> => - #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}}}}, + #{<<"schema">> => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{emqx_swagger_remote_schema, "ref2"}], {_, Components} = validate("/ref/remote", Spec, Refs), ExpectComponents = [ #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"properties">> => [ - {<<"page">>, #{description => <<"good page">>,example => 1, maximum => 100,minimum => 1,type => integer}}, - {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}}, + {<<"page">>, #{description => <<"good page">>,example => 1, + maximum => 100,minimum => 1,type => integer}}, + {<<"another_ref">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}}], <<"type">> => object}}, #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"properties">> => [ - {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>,type => string}}, - {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>,type => string}}], + {<<"ip">>, #{description => <<"IP:Port">>, + example => <<"127.0.0.1:80">>,type => string}}, + {<<"version">>, #{description => <<"a good version">>, + example => <<"1.0.0">>,type => string}}], <<"type">> => object}}], ?assertEqual(ExpectComponents, Components), ok. @@ -110,18 +127,22 @@ t_nest_ref(_Config) -> Spec = #{ post => #{parameters => [], requestBody => #{<<"content">> => #{<<"application/json">> => - #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.nest_ref">>}}}}, + #{<<"schema">> => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.nest_ref">>}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, nest_ref}], ExpectComponents = lists:sort([ #{<<"emqx_swagger_requestBody_SUITE.nest_ref">> => #{<<"properties">> => [ {<<"env">>, #{enum => [test,dev,prod],type => string}}, - {<<"another_ref">>, #{description => <<"nest ref">>, <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], + {<<"another_ref">>, #{description => <<"nest ref">>, + <<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}}], <<"type">> => object}}, #{<<"emqx_swagger_requestBody_SUITE.good_ref">> => #{<<"properties">> => [ - {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, example => <<"127.0.0.1:80">>,type => string}}, + {<<"webhook-host">>, #{default => <<"127.0.0.1:80">>, + example => <<"127.0.0.1:80">>,type => string}}, {<<"log_dir">>, #{example => <<"var/log/emqx">>,type => string}}, - {<<"tag">>, #{description => <<"tag">>, example => <<"binary-example">>,type => string}}], + {<<"tag">>, #{description => <<"tag">>, + example => <<"binary-example">>,type => string}}], <<"type">> => object}}]), {_, Components} = validate("/ref/nest/ref", Spec, Refs), ?assertEqual(ExpectComponents, Components), @@ -153,9 +174,14 @@ t_ref_array_with_key(_Config) -> #{<<"schema">> => #{required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [ - {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, - {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, - {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}} + {<<"per_page">>, #{description => <<"good per page desc">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, + #{enum => [infinity], type => string}]}}, + {<<"array_refs">>, #{items => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, + type => array}} ]}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, good_ref}], @@ -166,7 +192,8 @@ t_ref_array_without_key(_Config) -> Spec = #{ post => #{parameters => [], requestBody => #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => - #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}}}, + #{items => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>}, type => array}}}}, responses => #{<<"200">> => #{description => <<"ok">>}}}}, Refs = [{?MODULE, good_ref}], validate("/ref/array/without/key", Spec, Refs), @@ -190,7 +217,8 @@ t_api_spec(_Config) -> {ok, #{body := #{<<"timeout">> := <<"infinity">>}}}, trans_requestBody(Path, Body, Filter0)), - {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}), + {Spec1, _} = emqx_dashboard_swagger:spec(?MODULE, + #{check_schema => true, translate_body => true}), Filter1 = filter(Spec1, Path), ?assertMatch( {ok, #{body := #{<<"timeout">> := infinity}}}, @@ -237,7 +265,8 @@ t_object_notrans(_Config) -> <<"tag">> => <<"god_tag">> } }, - {ok, #{body := ActualBody}} = trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request/2), + {ok, #{body := ActualBody}} = trans_requestBody(Path, Body, + fun emqx_dashboard_swagger:filter_check_request/2), ?assertEqual(Body, ActualBody), ok. @@ -444,7 +473,8 @@ filter(ApiSpec, Path) -> Filter. trans_requestBody(Path, Body) -> - trans_requestBody(Path, Body, fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2). + trans_requestBody(Path, Body, + fun emqx_dashboard_swagger:filter_check_request_and_translate_body/2). trans_requestBody(Path, Body, Filter) -> Meta = #{module => ?MODULE, method => post, path => Path}, @@ -453,7 +483,8 @@ trans_requestBody(Path, Body, Filter) -> api_spec() -> emqx_dashboard_swagger:spec(?MODULE). paths() -> - ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/ref/array/with/key", "/ref/array/without/key"]. + ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", + "/ref/array/with/key", "/ref/array/without/key"]. schema("/object") -> to_schema([ diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index dff491225..4d4a9413e 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -40,9 +40,12 @@ t_object(_config) -> #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{required => [<<"timeout">>, <<"per_page">>], <<"properties">> => [ - {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, - {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, - {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}], + {<<"per_page">>, #{description => <<"good per page desc">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"timeout">>, #{default => 5, <<"oneOf">> => + [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, + {<<"inner_ref">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}], <<"type">> => object}}}}, ExpectRefs = [{?MODULE, good_ref}], validate(Path, Object, ExpectRefs), @@ -80,14 +83,17 @@ t_nest_object(_Config) -> Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [ - {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, + maximum => 100, minimum => 1, type => integer}}, {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, {<<"nest_object">>, #{<<"type">> => object, <<"properties">> => [ {<<"good_nest_1">>, #{example => 100, type => integer}}, - {<<"good_nest_2">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>} + {<<"good_nest_2">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>} }]}}, - {<<"inner_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}] + {<<"inner_ref">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}}] }}}}, ExpectRefs = [{?MODULE, good_ref}], validate(Path, Object, ExpectRefs), @@ -138,7 +144,8 @@ t_bad_ref(_Config) -> Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.bad_ref">>}}}}, ExpectRefs = [{?MODULE, bad_ref}], - ?assertThrow({error, #{module := ?MODULE, msg := <<"Object only supports not empty proplists">>}}, + ?assertThrow({error, #{module := ?MODULE, + msg := <<"Object only supports not empty proplists">>}}, validate(Path, Object, ExpectRefs)), ok. @@ -158,7 +165,8 @@ t_nest_ref(_Config) -> t_complicated_type(_Config) -> Path = "/ref/complicated_type", - Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{<<"properties">> => + Object = #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"properties">> => [ {<<"no_neg_integer">>, #{example => 100, minimum => 1, type => integer}}, {<<"url">>, #{example => <<"http://127.0.0.1">>, type => string}}, @@ -176,7 +184,8 @@ t_complicated_type(_Config) -> {<<"comma_separated_list">>, #{example => <<"item1,item2">>, type => string}}, {<<"comma_separated_atoms">>, #{example => <<"item1,item2">>, type => string}}, {<<"log_level">>, - #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all], type => string}}, + #{enum => [debug, info, notice, warning, error, critical, alert, emergency, all], + type => string}}, {<<"fix_integer">>, #{default => 100, enum => [100], example => 100,type => integer}} ], <<"type">> => object}}}}, @@ -192,15 +201,20 @@ t_ref_array_with_key(_Config) -> Path = "/ref/array/with/key", Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ required => [<<"timeout">>], <<"type">> => object, <<"properties">> => [ - {<<"per_page">>, #{description => <<"good per page desc">>, example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"per_page">>, #{description => <<"good per page desc">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, {<<"timeout">>, #{default => 5, <<"oneOf">> => [#{example => <<"1h">>, type => string}, #{enum => [infinity], type => string}]}}, {<<"assert">>, #{description => <<"money">>, example => 3.14159, type => number}}, - {<<"number_ex">>, #{description => <<"number example">>, example => 42, type => number}}, - {<<"percent_ex">>, #{description => <<"percent example">>, example => <<"12%">>, type => number}}, - {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, example => <<"32s">>, type => string}}, + {<<"number_ex">>, #{description => <<"number example">>, + example => 42, type => number}}, + {<<"percent_ex">>, #{description => <<"percent example">>, + example => <<"12%">>, type => number}}, + {<<"duration_ms_ex">>, #{description => <<"duration ms example">>, + example => <<"32s">>, type => string}}, {<<"atom_ex">>, #{description => <<"atom ex">>, example => atom, type => string}}, - {<<"array_refs">>, #{items => #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}} + {<<"array_refs">>, #{items => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_response_SUITE.good_ref">>}, type => array}} ]} }}}, ExpectRefs = [{?MODULE, good_ref}], @@ -227,25 +241,34 @@ t_hocon_schema_function(_Config) -> }}, #{<<"emqx_swagger_remote_schema.ref2">> => #{<<"type">> => object, <<"properties">> => [ - {<<"page">>, #{description => <<"good page">>, example => 1, maximum => 100, minimum => 1, type => integer}}, - {<<"another_ref">>, #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}} + {<<"page">>, #{description => <<"good page">>, + example => 1, maximum => 100, minimum => 1, type => integer}}, + {<<"another_ref">>, #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_remote_schema.ref3">>}} ] }}, #{<<"emqx_swagger_remote_schema.ref3">> => #{<<"type">> => object, <<"properties">> => [ - {<<"ip">>, #{description => <<"IP:Port">>, example => <<"127.0.0.1:80">>, type => string}}, - {<<"version">>, #{description => <<"a good version">>, example => <<"1.0.0">>, type => string}}] + {<<"ip">>, #{description => <<"IP:Port">>, + example => <<"127.0.0.1:80">>, type => string}}, + {<<"version">>, #{description => <<"a good version">>, + example => <<"1.0.0">>, type => string}}] }}, - #{<<"emqx_swagger_remote_schema.root">> => #{required => [<<"default_password">>, <<"default_username">>], + #{<<"emqx_swagger_remote_schema.root">> => + #{required => [<<"default_password">>, <<"default_username">>], <<"properties">> => [{<<"listeners">>, #{items => #{<<"oneOf">> => [#{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref2">>}, - #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, type => array}}, + #{<<"$ref">> => <<"#/components/schemas/emqx_swagger_remote_schema.ref1">>}]}, + type => array}}, {<<"default_username">>, #{default => <<"admin">>, example => <<"string-example">>, type => string}}, - {<<"default_password">>, #{default => <<"public">>, example => <<"string-example">>, type => string}}, - {<<"sample_interval">>, #{default => <<"10s">>, example => <<"1h">>, type => string}}, - {<<"token_expired_time">>, #{default => <<"30m">>, example => <<"12m">>, type => string}}], + {<<"default_password">>, + #{default => <<"public">>, example => <<"string-example">>, type => string}}, + {<<"sample_interval">>, + #{default => <<"10s">>, example => <<"1h">>, type => string}}, + {<<"token_expired_time">>, + #{default => <<"30m">>, example => <<"12m">>, type => string}}], <<"type">> => object}}], ExpectRefs = [{emqx_swagger_remote_schema, "root"}], {_, Components} = validate(Path, Object, ExpectRefs), diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index bce57e559..8bdf1bb3c 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -89,8 +89,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", @@ -118,8 +118,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) @@ -129,3 +130,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 0a540f4c1..82eb0b52c 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -59,12 +59,18 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> SslOpts -> [{ssl_options, SslOpts}] end, - _ = grpc:start_server(GwName, ListenOn, Services, SvrOptions), - ?ULOG("Start ~ts gRPC server on ~p successfully.~n", [GwName, ListenOn]). + case grpc:start_server(GwName, ListenOn, Services, SvrOptions) of + {ok, _SvrPid} -> + console_print("Start ~ts gRPC server on ~p successfully.", + [GwName, ListenOn]); + {error, Reason} -> + ?ELOG("Falied to start ~ts gRPC server on ~p, reason: ~p", + [GwName, ListenOn, Reason]) + end. stop_grpc_server(GwName) -> _ = grpc:stop_server(GwName), - ?ULOG("Stop ~s gRPC server successfully.~n", [GwName]). + console_print("Stop ~s gRPC server successfully.~n", [GwName]). start_grpc_client_channel(_GwName, undefined) -> undefined; @@ -157,8 +163,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", @@ -212,8 +218,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) @@ -223,3 +230,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index 899a7109f..6e01161bb 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -91,8 +91,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", @@ -131,8 +131,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) @@ -142,3 +143,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index 4a683ebdf..377c4f6d6 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -108,8 +108,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", @@ -142,8 +142,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) @@ -153,3 +154,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index de7321c92..41df189bc 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -93,8 +93,8 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Gateway ~ts:~ts:~ts on ~ts started.~n", - [GwName, Type, LisName, ListenOnStr]), + console_print("Gateway ~ts:~ts:~ts on ~ts started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n", @@ -127,8 +127,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Gateway ~ts:~ts:~ts on ~ts stopped.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> + console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) @@ -138,3 +139,9 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). + +-ifndef(TEST). +console_print(Fmt, Args) -> ?ULOG(Fmt, Args). +-else. +console_print(_Fmt, _Args) -> ok. +-endif. diff --git a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl index db6334565..d634b7898 100644 --- a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl @@ -40,7 +40,7 @@ , on_received_messages/2 ]). --define(LOG(Fmt, Args), io:format(standard_error, Fmt, Args)). +-define(LOG(Fmt, Args), ct:pal(Fmt, Args)). -define(HTTP, #{grpc_opts => #{service_protos => [emqx_exproto_pb], services => #{'emqx.exproto.v1.ConnectionHandler' => ?MODULE}}, @@ -188,13 +188,15 @@ on_received_messages(Stream, _Md) -> %%-------------------------------------------------------------------- handle_in(Conn, ?TYPE_CONNECT, #{<<"clientinfo">> := ClientInfo, <<"password">> := Password}) -> - NClientInfo = maps:from_list([{binary_to_atom(K, utf8), V} || {K, V} <- maps:to_list(ClientInfo)]), + NClientInfo = maps:from_list( + [{binary_to_atom(K, utf8), V} + || {K, V} <- maps:to_list(ClientInfo)]), case ?authenticate(#{conn => Conn, clientinfo => NClientInfo, password => Password}) of {ok, #{code := 'SUCCESS'}, _} -> case maps:get(keepalive, NClientInfo, 0) of 0 -> ok; Intv -> - io:format("Try call start_timer with ~ps", [Intv]), + ?LOG("Try call start_timer with ~ps", [Intv]), ?start_timer(#{conn => Conn, type => 'KEEPALIVE', interval => Intv}) end, handle_out(Conn, ?TYPE_CONNACK, 0); diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 831b0d72f..c637f44c2 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -36,7 +36,7 @@ -define(FLAG_RETAIN(X),X). -define(FLAG_SESSION(X),X). --define(LOG(Format, Args), ct:print("TEST: " ++ Format, Args)). +-define(LOG(Format, Args), ct:pal("TEST: " ++ Format, Args)). -define(MAX_PRED_TOPIC_ID, 2). -define(PREDEF_TOPIC_ID1, 1). @@ -51,6 +51,8 @@ -define(CLIENTID, iolist_to_binary([atom_to_list(?FUNCTION_NAME), "-", integer_to_list(erlang:system_time())])). +-elvis([{elvis_style, dont_repeat_yourself, disable}]). + -define(CONF_DEFAULT, <<" gateway.mqttsn { gateway_id = 1 @@ -177,7 +179,7 @@ t_subscribe_case02(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), @@ -227,8 +229,14 @@ t_subscribe_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -%%In this case We use predefined topic name to register and subcribe, and expect to receive the corresponding predefined topic id but not a new generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal topics. Once we give more restrictions to different topic id type, this case would be deleted or modified. +%% In this case We use predefined topic name to register and subcribe, +%% and expect to receive the corresponding predefined topic id but not a new +%% generated topic id from broker. We design this case to illustrate +%% emqx_sn_gateway's compatibility of dealing with predefined and normal +%% topics. +%% +%% Once we give more restrictions to different topic id type, this case +%% would be deleted or modified. t_subscribe_case04(_) -> Dup = 0, QoS = 0, @@ -236,7 +244,7 @@ t_subscribe_case04(_) -> Will = 0, CleanSession = 0, MsgId = 1, - TopicId = ?PREDEF_TOPIC_ID1, %this TopicId is the predefined topic id corresponding to ?PREDEF_TOPIC_NAME1 + TopicId = ?PREDEF_TOPIC_ID1, ReturnCode = 0, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, @@ -246,8 +254,12 @@ t_subscribe_case04(_) -> send_register_msg(Socket, Topic1, MsgId), ?assertEqual(<<7, ?SN_REGACK, TopicId:16, MsgId:16, 0:8>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, Topic1, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -277,21 +289,36 @@ t_subscribe_case05(_) -> ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"abcD">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/sport/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/a/+/water">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_subscribe_msg_normal_topic(Socket, QoS, <<"/Tom/Home">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId2:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId2:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, <<"abcD">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -316,19 +343,37 @@ t_subscribe_case06(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_register_msg(Socket, <<"abc">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual( + <<7, ?SN_REGACK, TopicId1:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/#">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual( + <<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/blue/+/white">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId0:16, MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, receive_response(Socket)), + ?assertEqual( + <<7, ?SN_REGACK, TopicId0:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED:8>>, + receive_response(Socket) + ), send_register_msg(Socket, <<"/$sys/rain">>, MsgId), - ?assertEqual(<<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, receive_response(Socket)), + ?assertEqual( + <<7, ?SN_REGACK, TopicId2:16, MsgId:16, 0:8>>, + receive_response(Socket) + ), send_subscribe_msg_short_topic(Socket, QoS, <<"Q2">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ReturnCode>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, + CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ReturnCode>>, + receive_response(Socket) + ), send_unsubscribe_msg_normal_topic(Socket, <<"Q2">>, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -352,8 +397,12 @@ t_subscribe_case07(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket) + ), send_unsubscribe_msg_predefined_topic(Socket, TopicId2, MsgId), ?assertEqual(<<4, ?SN_UNSUBACK, MsgId:16>>, receive_response(Socket)), @@ -375,8 +424,12 @@ t_subscribe_case08(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_reserved_topic(Socket, QoS, TopicId2, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + ?SN_INVALID_TOPIC_ID:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -400,15 +453,21 @@ t_publish_negqos_case09(_) -> send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, NegQoS, MsgId1, TopicId1, Payload1), timer:sleep(100), case ?ENABLE_QOS3 of true -> - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What) end, @@ -441,7 +500,9 @@ t_publish_qos0_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -463,15 +524,21 @@ t_publish_qos0_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, PredefTopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -494,15 +561,21 @@ t_publish_qos0_case3(_) -> Topic = <<"/a/b/c">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId1, TopicId, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -524,8 +597,12 @@ t_publish_qos0_case04(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 2, Payload1 = <<20, 21, 22, 23>>, @@ -533,7 +610,9 @@ t_publish_qos0_case04(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId1, Topic, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + Topic/binary, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -554,8 +633,12 @@ t_publish_qos0_case05(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -577,15 +660,21 @@ t_publish_qos0_case06(_) -> Topic = <<"abc">>, send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), MsgId1 = 3, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId1, TopicId1, Payload1), timer:sleep(100), - Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, + Eexp = <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, (mid(0)):16, <<20, 21, 22, 23>>/binary>>, What = receive_response(Socket), ?assertEqual(Eexp, What), @@ -607,16 +696,24 @@ t_publish_qos1_case01(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId1:16, + MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), timer:sleep(100), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), send_disconnect_msg(Socket, undefined), gen_udp:close(Socket). @@ -635,12 +732,17 @@ t_publish_qos1_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), - ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket)), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -655,7 +757,8 @@ t_publish_qos1_case03(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_publish_msg_predefined_topic(Socket, QoS, MsgId, tid(5), <<20, 21, 22, 23>>), - ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicId5:16, MsgId:16, ?SN_RC_INVALID_TOPIC_ID>>, + receive_response(Socket)), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -674,15 +777,19 @@ t_publish_qos1_case04(_) -> send_connect_msg(Socket, ClientId), ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_short_topic(Socket, QoS, <<"ab">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, - ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Topic = <<"ab">>, Payload1 = <<20, 21, 22, 23>>, send_publish_msg_short_topic(Socket, QoS, MsgId, Topic, Payload1), <> = Topic, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket)), timer:sleep(100), send_disconnect_msg(Socket, undefined), @@ -708,7 +815,9 @@ t_publish_qos1_case05(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/#">>, <<20, 21, 22, 23>>), <> = <<"/#">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, + receive_response(Socket)), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -734,7 +843,8 @@ t_publish_qos1_case06(_) -> send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/+">>, <<20, 21, 22, 23>>), <> = <<"/+">>, - ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), + ?assertEqual(<<7, ?SN_PUBACK, TopicIdShort:16, + MsgId:16, ?SN_RC_NOT_SUPPORTED>>, receive_response(Socket)), send_disconnect_msg(Socket, undefined), ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), @@ -761,7 +871,12 @@ t_publish_qos2_case01(_) -> send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual( + <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId1:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -783,15 +898,23 @@ t_publish_qos2_case02(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_predefined_topic(Socket, QoS, PredefTopicId, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, ?FNU:1, QoS:2, + ?FNU:5, PredefTopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_predefined_topic(Socket, QoS, MsgId, PredefTopicId, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC :2, PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual( + <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_PREDEFINED_TOPIC:2, + PredefTopicId:16, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -814,15 +937,23 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), send_subscribe_msg_normal_topic(Socket, QoS, <<"/#">>, MsgId), - ?assertEqual(<<8, ?SN_SUBACK, ?FNU:1, QoS:2, ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, - receive_response(Socket)), + ?assertEqual( + <<8, ?SN_SUBACK, ?FNU:1, QoS:2, + ?FNU:5, TopicId0:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket) + ), Payload1 = <<20, 21, 22, 23>>, send_publish_msg_short_topic(Socket, QoS, MsgId, <<"/a">>, Payload1), ?assertEqual(<<4, ?SN_PUBREC, MsgId:16>>, receive_response(Socket)), send_pubrel_msg(Socket, MsgId), - ?assertEqual(<<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_SHORT_TOPIC :2, <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, receive_response(Socket)), + ?assertEqual( + <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_SHORT_TOPIC:2, + <<"/a">>/binary, 1:16, <<20, 21, 22, 23>>/binary>>, + receive_response(Socket) + ), ?assertEqual(<<4, ?SN_PUBCOMP, MsgId:16>>, receive_response(Socket)), timer:sleep(100), @@ -1095,7 +1226,12 @@ t_will_case06(_) -> % send_register_msg(Socket, TopicName1, MsgId1), % ?assertEqual(<<7, ?SN_REGACK, TopicId1:16, MsgId1:16, 0:8>>, receive_response(Socket)), % send_subscribe_msg_predefined_topic(Socket, QoS, TopicId1, MsgId), -% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId1:16, MsgId:16, ReturnCode>>, receive_response(Socket)), +% ?assertEqual( +% <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, +% WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, +% TopicId1:16, MsgId:16, ReturnCode>>, +% receive_response(Socket) +% ), % % % goto asleep state % send_disconnect_msg(Socket, 1), @@ -1121,7 +1257,9 @@ t_will_case06(_) -> % % %% the broker should sent dl msgs to the awake client before sending the pingresp % UdpData = receive_response(Socket), -% MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), +% MsgId_udp = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicId1, Payload1}, UdpData), % send_puback_msg(Socket, TopicId1, MsgId_udp), % % %% check the pingresp is received at last @@ -1153,8 +1291,12 @@ t_will_case06(_) -> % CleanSession = 0, % ReturnCode = 0, % send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), -% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, -% receive_response(Socket)), +% ?assertEqual( +% <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, +% WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, +% TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket) +% ), % % % goto asleep state % send_disconnect_msg(Socket, 1), @@ -1188,11 +1330,15 @@ t_will_case06(_) -> % send_regack_msg(Socket, TopicIdNew, MsgId3), % % UdpData2 = receive_response(Socket), -% MsgId_udp2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), +% MsgId_udp2 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload1}, UdpData2), % send_puback_msg(Socket, TopicIdNew, MsgId_udp2), % % UdpData3 = receive_response(Socket), -% MsgId_udp3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), +% MsgId_udp3 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData3), % send_puback_msg(Socket, TopicIdNew, MsgId_udp3), % % ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), @@ -1228,8 +1374,12 @@ receive_publish(Socket) -> % CleanSession = 0, % ReturnCode = 0, % send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), -% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, -% receive_response(Socket)), +% ?assertEqual( +% <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, +% WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, +% TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket) +% ), % % % goto asleep state % SleepDuration = 30, @@ -1262,12 +1412,16 @@ receive_publish(Socket) -> % send_regack_msg(Socket, TopicIdNew, MsgId_reg), % % UdpData2 = receive_response(Socket), -% MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), +% MsgId2 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), % send_puback_msg(Socket, TopicIdNew, MsgId2), % timer:sleep(50), % % UdpData3 = wrap_receive_response(Socket), -% MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), +% MsgId3 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), % send_puback_msg(Socket, TopicIdNew, MsgId3), % timer:sleep(50), % @@ -1334,7 +1488,9 @@ receive_publish(Socket) -> % send_pingreq_msg(Socket, ClientId), % % UdpData = wrap_receive_response(Socket), -% MsgId_udp = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), +% MsgId_udp = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicId_tom, Payload1}, UdpData), % send_pubrec_msg(Socket, MsgId_udp), % ?assertMatch(<<_:8, ?SN_PUBREL:8, _/binary>>, receive_response(Socket)), % send_pubcomp_msg(Socket, MsgId_udp), @@ -1369,8 +1525,12 @@ receive_publish(Socket) -> % send_register_msg(Socket, TopicName_tom, MsgId1), % TopicId_tom = check_regack_msg_on_udp(MsgId1, receive_response(Socket)), % send_subscribe_msg_predefined_topic(Socket, QoS, TopicId_tom, MsgId1), -% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId_tom:16, MsgId1:16, ReturnCode>>, -% receive_response(Socket)), +% ?assertEqual( +% <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, +% WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, +% TopicId_tom:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket) +% ), % % % goto asleep state % send_disconnect_msg(Socket, SleepDuration), @@ -1448,8 +1608,12 @@ receive_publish(Socket) -> % CleanSession = 0, % ReturnCode = 0, % send_subscribe_msg_normal_topic(Socket, QoS, TopicName1, MsgId1), -% ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, TopicId0:16, MsgId1:16, ReturnCode>>, -% receive_response(Socket)), +% ?assertEqual( +% <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, +% WillBit:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, +% TopicId0:16, MsgId1:16, ReturnCode>>, +% receive_response(Socket) +% ), % % goto asleep state % SleepDuration = 30, % send_disconnect_msg(Socket, SleepDuration), @@ -1483,7 +1647,9 @@ receive_publish(Socket) -> % udp_receive_timeout -> % ok; % UdpData2 -> -% MsgId2 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), +% MsgId2 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload2}, UdpData2), % send_puback_msg(Socket, TopicIdNew, MsgId2) % end, % timer:sleep(100), @@ -1492,7 +1658,9 @@ receive_publish(Socket) -> % udp_receive_timeout -> % ok; % UdpData3 -> -% MsgId3 = check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), +% MsgId3 = check_publish_msg_on_udp( +% {Dup, QoS, Retain, WillBit, CleanSession, +% ?SN_NORMAL_TOPIC, TopicIdNew, Payload3}, UdpData3), % send_puback_msg(Socket, TopicIdNew, MsgId3) % end, % timer:sleep(100), @@ -1510,7 +1678,8 @@ receive_publish(Socket) -> % % %% send PINGREQ again to enter awake state % send_pingreq_msg(Socket, ClientId), -% %% will not receive any buffered PUBLISH messages buffered before last awake, only receive PINGRESP here +% %% will not receive any buffered PUBLISH messages buffered before last +% %% awake, only receive PINGRESP here % ?assertEqual(<<2, ?SN_PINGRESP>>, receive_response(Socket)), % % gen_udp:close(Socket). @@ -1913,8 +2082,11 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket PubMsg = receive_response(Socket), Length = 7 + byte_size(Payload), ?LOG("check_dispatched_message ~p~n", [PubMsg]), - ?LOG("expected ~p xx ~p~n", [<>, Payload]), - <> = PubMsg, + ?LOG("expected ~p xx ~p~n", + [<>, Payload]), + <> = PubMsg, case QoS of 0 -> ok; 1 -> send_puback_msg(Socket, TopicId, MsgId); @@ -1926,11 +2098,13 @@ check_dispatched_message(Dup, QoS, Retain, TopicIdType, TopicId, Payload, Socket get_udp_broadcast_address() -> "255.255.255.255". -check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, TopicType, TopicId, Payload}, UdpData) -> +check_publish_msg_on_udp({Dup, QoS, Retain, WillBit, CleanSession, + TopicType, TopicId, Payload}, UdpData) -> <> = UdpData, ct:pal("UdpData: ~p, Payload: ~p, PayloadIn: ~p", [UdpData, Payload, PayloadIn]), Size9 = byte_size(Payload) + 7, - Eexp = <>, + Eexp = <>, ?assertEqual(Eexp, HeaderUdp), % mqtt-sn header should be same ?assertEqual(Payload, PayloadIn), % payload should be same MsgId. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index a920b18ad..af05bd8a3 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -148,7 +148,6 @@ init_per_testcase(t_events, Config) -> #{id => <<"rule:t_events">>, sql => SQL, outputs => [ - #{function => console}, #{function => <<"emqx_rule_engine_SUITE:output_record_triggered_events">>, args => #{}} ], diff --git a/bin/emqx b/bin/emqx index 63c09cc41..e1b72d1f9 100755 --- a/bin/emqx +++ b/bin/emqx @@ -587,6 +587,7 @@ case "${COMMAND}" in ping) assert_node_alive + echo pong ;; escript) diff --git a/build b/build index 6814efe02..6ba943fab 100755 --- a/build +++ b/build @@ -52,13 +52,10 @@ log() { } docgen() { - local conf_doc_html libs_dir1 libs_dir2 - conf_doc_html="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.html" - echo "===< Generating config document $conf_doc_html" + local libs_dir1 libs_dir2 libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)" libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" - # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 -eval "file:write_file('$conf_doc_html', hocon_schema_html:gen(emqx_conf_schema, \"EMQ X ${PKG_VSN}\")), halt(0)." + local conf_doc_markdown conf_doc_markdown="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.md" echo "===< Generating config document $conf_doc_markdown" @@ -146,6 +143,7 @@ make_zip() { ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" (cd "${tard}" && zip -qr - emqx) > "${zipball}" + log "Zip package successfully created: ${zipball}" } ## This function builds the default docker image based on alpine:3.14 (by default) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 1dd7d318a..5538c5a0d 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -28,6 +28,7 @@ RUN cd /emqx \ FROM $RUN_FROM +## define ARG again after 'FROM $RUN_FROM' ARG EMQX_NAME=emqx COPY deploy/docker/docker-entrypoint.sh /usr/bin/ diff --git a/rebar.config.erl b/rebar.config.erl index 78e4f3e41..22949f233 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -148,44 +148,52 @@ prod_overrides() -> profiles() -> Vsn = get_vsn(), - [ {'emqx', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, bin, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, pkg, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - ]} - , {'emqx-ee', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, bin, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, cloud, pkg, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, edge, bin, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} - , {relx, relx(Vsn, edge, pkg, ce)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ce)} - ]} - , {check, [ {erl_opts, common_compile_opts()} - , {project_app_dirs, project_app_dirs(ce)} - ]} - , {test, [ {deps, test_deps()} - , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) } - , {extra_src_dirs, [{"test", [{recursive, true}]}]} - , {project_app_dirs, project_app_dirs(ce)} - ]} + [ {'emqx', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, bin, ce)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ce)} + ]} + , {'emqx-pkg', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, pkg, ce)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ce)} + ]} + , {'emqx-enterprise', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, bin, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-enterprise-pkg', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, pkg, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-edge', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, edge, bin, ce)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ce)} + ]} + , {'emqx-edge-pkg', + [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, edge, pkg, ce)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ce)} + ]} + , {check, + [ {erl_opts, common_compile_opts()} + , {project_app_dirs, project_app_dirs(ce)} + ]} + , {test, + [ {deps, test_deps()} + , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) } + , {extra_src_dirs, [{"test", [{recursive, true}]}]} + , {project_app_dirs, project_app_dirs(ce)} + ]} ]. %% RelType: cloud (full size) | edge (slim size) diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 39a71b492..8c420c1bd 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -154,7 +154,7 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), Script = "mkdir -p ${OUTFILE} && - wget -O ${OUTFILE}.zip ${URL} && + wget -c -O ${OUTFILE}.zip ${URL} && unzip -n -d ${OUTFILE} ${OUTFILE}.zip", Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], bash(Script, Env), @@ -298,12 +298,15 @@ render_appfile(File, Upgrade, Downgrade) -> ok = file:write_file(File, IOList). create_stub(App) -> - case locate(src, App, ".app.src") of + Ext = ".app.src", + case locate(src, App, Ext) of {ok, AppSrc} -> - AppupFile = filename:basename(AppSrc) ++ ".appup.src", + DirName = filename:dirname(AppSrc), + AppupFile = filename:basename(AppSrc, Ext) ++ ".appup.src", Default = {<<".*">>, []}, - render_appfile(AppupFile, [Default], [Default]), - AppupFile; + AppupFileFullpath = filename:join(DirName, AppupFile), + render_appfile(AppupFileFullpath, [Default], [Default]), + {ok, AppupFileFullpath}; undefined -> false end.