From e095de7367cd09892f527d6d4389b81c05b02019 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 16 Oct 2023 14:08:12 +0800 Subject: [PATCH 001/111] feat(api_key): add RBAC feature for the API key --- apps/emqx/test/emqx_common_test_http.erl | 8 +- .../emqx_dashboard/include/emqx_dashboard.hrl | 11 +- .../include/emqx_dashboard_rbac.hrl | 33 ++++ apps/emqx_dashboard/src/emqx_dashboard.erl | 5 +- .../src/emqx_dashboard_admin.erl | 6 +- .../src/emqx_dashboard_rbac.app.src | 2 +- .../src/emqx_dashboard_rbac.erl | 54 +++++-- .../src/emqx_mgmt_api_api_keys.erl | 40 ++++- apps/emqx_management/src/emqx_mgmt_auth.erl | 152 +++++++++++++----- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 31 ++-- rel/i18n/emqx_mgmt_api_api_keys.hocon | 3 + 11 files changed, 261 insertions(+), 84 deletions(-) create mode 100644 apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 7f50db92b..1034e310f 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -17,6 +17,7 @@ -module(emqx_common_test_http). -include_lib("common_test/include/ct.hrl"). +-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl"). -export([ request_api/3, @@ -90,7 +91,12 @@ create_default_app() -> Now = erlang:system_time(second), ExpiredAt = Now + timer:minutes(10), emqx_mgmt_auth:create( - ?DEFAULT_APP_ID, ?DEFAULT_APP_SECRET, true, ExpiredAt, <<"default app key for test">> + ?DEFAULT_APP_ID, + ?DEFAULT_APP_SECRET, + true, + ExpiredAt, + <<"default app key for test">>, + ?ROLE_API_SUPERUSER ). delete_default_app() -> diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index 9013436e7..c41dbb71c 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -13,16 +13,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --define(ADMIN, emqx_admin). +-include("emqx_dashboard_rbac.hrl"). -%% TODO: -%% The predefined roles of the preliminary RBAC implementation, -%% these may be removed when developing the full RBAC feature. -%% In full RBAC feature, the role may be customised created and deleted, -%% a predefined configuration would replace these macros. --define(ROLE_VIEWER, <<"viewer">>). --define(ROLE_SUPERUSER, <<"administrator">>). --define(ROLE_DEFAULT, ?ROLE_SUPERUSER). +-define(ADMIN, emqx_admin). -define(BACKEND_LOCAL, local). -define(SSO_USERNAME(Backend, Name), {Backend, Name}). diff --git a/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl new file mode 100644 index 000000000..8f49464a4 --- /dev/null +++ b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_DASHBOARD_RBAC). +-define(EMQX_DASHBOARD_RBAC, true). + +%% TODO: +%% The predefined roles of the preliminary RBAC implementation, +%% these may be removed when developing the full RBAC feature. +%% In full RBAC feature, the role may be customised created and deleted, +%% a predefined configuration would replace these macros. +-define(ROLE_VIEWER, <<"viewer">>). +-define(ROLE_SUPERUSER, <<"administrator">>). +-define(ROLE_DEFAULT, ?ROLE_SUPERUSER). + +-define(ROLE_API_VIEWER, <<"api_viewer">>). +-define(ROLE_API_SUPERUSER, <<"api_administrator">>). +-define(ROLE_API_PUBLISHER, <<"api_publisher">>). +-define(ROLE_API_DEFAULT, ?ROLE_API_SUPERUSER). + +-endif. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 4f9e34238..fbd801410 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -251,7 +251,7 @@ listeners() -> api_key_authorize(Req, Key, Secret) -> Path = cowboy_req:path(Req), - case emqx_mgmt_auth:authorize(Path, Key, Secret) of + case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of ok -> {ok, #{auth_type => api_key, api_key => Key}}; {error, <<"not_allowed">>} -> @@ -259,6 +259,9 @@ api_key_authorize(Req, Key, Secret) -> ?BAD_API_KEY_OR_SECRET, <<"Not allowed, Check api_key/api_secret">> ); + {error, unauthorized_role} -> + {403, 'UNAUTHORIZED_ROLE', + <<"This API Key don't have permission to access this resource">>}; {error, _} -> return_unauthorized( ?BAD_API_KEY_OR_SECRET, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index e9aac164b..e2bde51bd 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -416,7 +416,7 @@ ensure_role(Role) when is_binary(Role) -> -if(?EMQX_RELEASE_EDITION == ee). legal_role(Role) -> - emqx_dashboard_rbac:valid_role(Role). + emqx_dashboard_rbac:valid_dashboard_role(Role). role(Data) -> emqx_dashboard_rbac:role(Data). @@ -447,8 +447,10 @@ lookup_user(Backend, Username) when is_atom(Backend) -> -dialyzer({no_match, [add_user/4, update_user/3]}). +legal_role(?ROLE_DEFAULT) -> + ok; legal_role(_) -> - ok. + {error, <<"Role does not exist">>}. role(_) -> ?ROLE_DEFAULT. diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src index 190764e2f..ec8e6cd3f 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_rbac, [ {description, "EMQX Dashboard RBAC"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index 28bd8960e..2bc6a5bf9 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -6,7 +6,12 @@ -include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). --export([check_rbac/2, role/1, valid_role/1]). +-export([ + check_rbac/2, + role/1, + valid_dashboard_role/1, + valid_api_role/1 +]). -dialyzer({nowarn_function, role/1}). %%===================================================================== @@ -31,25 +36,44 @@ role(#?ADMIN{role = Role}) -> role([]) -> ?ROLE_SUPERUSER; role(#{role := Role}) -> + Role; +role(Role) when is_binary(Role) -> Role. -valid_role(Role) -> - case lists:member(Role, role_list()) of +valid_dashboard_role(Role) -> + valid_role(dashboard, Role). + +valid_api_role(Role) -> + valid_role(api, Role). + +%% =================================================================== +check_rbac(?ROLE_SUPERUSER, _, _) -> + true; +check_rbac(?ROLE_API_SUPERUSER, _, _) -> + true; +check_rbac(?ROLE_VIEWER, <<"GET">>, _) -> + true; +check_rbac(?ROLE_API_VIEWER, <<"GET">>, _) -> + true; +%% this API is a special case +check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) -> + true; +check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>) -> + true; +check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>) -> + true; +check_rbac(_, _, _) -> + false. + +valid_role(Type, Role) -> + case lists:member(Role, role_list(Type)) of true -> ok; _ -> {error, <<"Role does not exist">>} end. -%% =================================================================== -check_rbac(?ROLE_SUPERUSER, _, _) -> - true; -check_rbac(?ROLE_VIEWER, <<"GET">>, _) -> - true; -%% this API is a special case -check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) -> - true; -check_rbac(_, _, _) -> - false. -role_list() -> - [?ROLE_VIEWER, ?ROLE_SUPERUSER]. +role_list(dashboard) -> + [?ROLE_VIEWER, ?ROLE_SUPERUSER]; +role_list(api) -> + [?ROLE_API_VIEWER, ?ROLE_API_PUBLISHER, ?ROLE_API_SUPERUSER]. diff --git a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl index 78bbef540..0523fd244 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_api_keys.erl @@ -19,6 +19,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl"). -export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]). -export([api_key/2, api_key_by_name/2]). @@ -150,7 +151,7 @@ fields(app) -> )}, {enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", required => false})}, {expired, hoconsc:mk(boolean(), #{desc => "Expired", required => false})} - ]; + ] ++ app_extend_fields(); fields(name) -> [ {name, @@ -192,7 +193,8 @@ api_key(post, #{body := App}) -> } = App, ExpiredAt = ensure_expired_at(App), Desc = unicode:characters_to_binary(Desc0, unicode), - case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of + Role = maps:get(<<"role">>, App, ?ROLE_API_DEFAULT), + case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc, Role) of {ok, NewApp} -> {200, emqx_mgmt_auth:format(NewApp)}; {error, Reason} -> @@ -218,10 +220,38 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) -> Enable = maps:get(<<"enable">>, Body, undefined), ExpiredAt = ensure_expired_at(Body), Desc = maps:get(<<"desc">>, Body, undefined), - case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc) of - {ok, App} -> {200, emqx_mgmt_auth:format(App)}; - {error, not_found} -> {404, ?NOT_FOUND_RESPONSE} + Role = maps:get(<<"role">>, Body, ?ROLE_API_DEFAULT), + case emqx_mgmt_auth:update(Name, Enable, ExpiredAt, Desc, Role) of + {ok, App} -> + {200, emqx_mgmt_auth:format(App)}; + {error, not_found} -> + {404, ?NOT_FOUND_RESPONSE}; + {error, Reason} -> + {400, #{ + code => 'BAD_REQUEST', + message => iolist_to_binary(io_lib:format("~p", [Reason])) + }} end. ensure_expired_at(#{<<"expired_at">> := ExpiredAt}) when is_integer(ExpiredAt) -> ExpiredAt; ensure_expired_at(_) -> infinity. + +-if(?EMQX_RELEASE_EDITION == ee). + +app_extend_fields() -> + [ + {role, + hoconsc:mk(binary(), #{ + desc => ?DESC(role), + default => ?ROLE_API_DEFAULT, + example => ?ROLE_API_DEFAULT, + validator => fun emqx_dashboard_rbac:valid_api_role/1 + })} + ]. + +-else. + +app_extend_fields() -> + []. + +-endif. diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 3d32afc19..c39e3888d 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -16,6 +16,7 @@ -module(emqx_mgmt_auth). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl"). -behaviour(emqx_db_backup). @@ -25,16 +26,16 @@ -behaviour(emqx_config_handler). -export([ - create/4, + create/5, read/1, - update/4, + update/5, delete/1, list/0, init_bootstrap_file/0, format/1 ]). --export([authorize/3]). +-export([authorize/4]). -export([post_config_update/5]). -export([backup_tables/0]). @@ -48,10 +49,11 @@ ]). -ifdef(TEST). --export([create/5]). +-export([create/6]). -endif. -define(APP, emqx_app). +-type api_user_role() :: binary(). -record(?APP, { name = <<>> :: binary() | '_', @@ -60,17 +62,21 @@ enable = true :: boolean() | '_', desc = <<>> :: binary() | '_', expired_at = 0 :: integer() | undefined | infinity | '_', - created_at = 0 :: integer() | '_' + created_at = 0 :: integer() | '_', + role = ?ROLE_DEFAULT :: api_user_role() | '_', + extra = #{} :: map() | '_' }). mnesia(boot) -> + Fields = record_info(fields, ?APP), ok = mria:create_table(?APP, [ {type, set}, {rlog_shard, ?COMMON_SHARD}, {storage, disc_copies}, {record_name, ?APP}, - {attributes, record_info(fields, ?APP)} - ]). + {attributes, Fields} + ]), + maybe_migrate_table(Fields). %%-------------------------------------------------------------------- %% Data backup @@ -95,13 +101,13 @@ init_bootstrap_file() -> ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}), init_bootstrap_file(File). -create(Name, Enable, ExpiredAt, Desc) -> +create(Name, Enable, ExpiredAt, Desc, Role) -> ApiSecret = generate_api_secret(), - create(Name, ApiSecret, Enable, ExpiredAt, Desc). + create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role). -create(Name, ApiSecret, Enable, ExpiredAt, Desc) -> +create(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) -> case mnesia:table_info(?APP, size) < 100 of - true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc); + true -> create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role); false -> {error, "Maximum ApiKey"} end. @@ -111,8 +117,13 @@ read(Name) -> [] -> {error, not_found} end. -update(Name, Enable, ExpiredAt, Desc) -> - trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]). +update(Name, Enable, ExpiredAt, Desc, Role) -> + case valid_role(Role) of + ok -> + trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]); + Error -> + Error + end. do_update(Name, Enable, ExpiredAt, Desc) -> case mnesia:read(?APP, Name, write) of @@ -138,37 +149,37 @@ do_delete(Name) -> [_App] -> mnesia:delete({?APP, Name}) end. -format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) -> - ExpiredAt = - case ExpiredAt0 of - infinity -> <<"infinity">>; - _ -> emqx_utils_calendar:epoch_to_rfc3339(ExpiredAt0, second) - end, - App#{ - expired_at => ExpiredAt, - created_at => emqx_utils_calendar:epoch_to_rfc3339(CreateAt, second) - }. +format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) -> + format_app_extend(App#{ + expired_at => format_epoch(ExpiredAt), + created_at => format_epoch(CreateAt) + }). + +format_epoch(infinity) -> + <<"infinity">>; +format_epoch(Epoch) -> + emqx_utils_calendar:epoch_to_rfc3339(Epoch, second). list() -> to_map(ets:match_object(?APP, #?APP{_ = '_'})). -authorize(<<"/api/v5/users", _/binary>>, _ApiKey, _ApiSecret) -> +authorize(<<"/api/v5/users", _/binary>>, _Req, _ApiKey, _ApiSecret) -> {error, <<"not_allowed">>}; -authorize(<<"/api/v5/api_key", _/binary>>, _ApiKey, _ApiSecret) -> +authorize(<<"/api/v5/api_key", _/binary>>, _Req, _ApiKey, _ApiSecret) -> {error, <<"not_allowed">>}; -authorize(<<"/api/v5/logout", _/binary>>, _ApiKey, _ApiSecret) -> +authorize(<<"/api/v5/logout", _/binary>>, _Req, _ApiKey, _ApiSecret) -> {error, <<"not_allowed">>}; -authorize(_Path, ApiKey, ApiSecret) -> +authorize(_Path, Req, ApiKey, ApiSecret) -> Now = erlang:system_time(second), case find_by_api_key(ApiKey) of - {ok, true, ExpiredAt, SecretHash} when ExpiredAt >= Now -> + {ok, true, ExpiredAt, SecretHash, Role} when ExpiredAt >= Now -> case emqx_dashboard_admin:verify_hash(ApiSecret, SecretHash) of - ok -> ok; + ok -> check_rbac(Req, Role); error -> {error, "secret_error"} end; - {ok, true, _ExpiredAt, _SecretHash} -> + {ok, true, _ExpiredAt, _SecretHash, _Role} -> {error, "secret_expired"}; - {ok, false, _ExpiredAt, _SecretHash} -> + {ok, false, _ExpiredAt, _SecretHash, _Role} -> {error, "secret_disable"}; {error, Reason} -> {error, Reason} @@ -177,8 +188,12 @@ authorize(_Path, ApiKey, ApiSecret) -> find_by_api_key(ApiKey) -> Fun = fun() -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end, case mria:ro_transaction(?COMMON_SHARD, Fun) of - {atomic, [#?APP{api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt}]} -> - {ok, Enable, ExpiredAt, SecretHash}; + {atomic, [ + #?APP{ + api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt, role = Role + } + ]} -> + {ok, Enable, ExpiredAt, SecretHash, Role}; _ -> {error, "not_found"} end. @@ -202,7 +217,7 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT is_expired(undefined) -> false; is_expired(ExpiredTime) -> ExpiredTime < erlang:system_time(second). -create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> +create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) -> App = #?APP{ name = Name, @@ -211,7 +226,8 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> desc = Desc, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_utils:gen_id(16)) + api_key = list_to_binary(emqx_utils:gen_id(16)), + role = Role }, case create_app(App) of {ok, Res} -> @@ -220,8 +236,13 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> Error end. -create_app(App = #?APP{api_key = ApiKey, name = Name}) -> - trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]). +create_app(App = #?APP{api_key = ApiKey, name = Name, role = Role}) -> + case valid_role(Role) of + ok -> + trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]); + Error -> + Error + end. force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) -> trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]). @@ -340,3 +361,60 @@ add_bootstrap_file(File, Dev, MP, Line) -> {error, Reason} -> throw(#{file => File, line => Line, reason => Reason}) end. + +-if(?EMQX_RELEASE_EDITION == ee). +check_rbac(Req, Role) -> + case emqx_dashboard_rbac:check_rbac(Req, Role) of + true -> + ok; + _ -> + {error, unauthorized_role} + end. + +format_app_extend(App) -> + App. + +valid_role(Role) -> + emqx_dashboard_rbac:valid_api_role(Role). + +-else. + +check_rbac(_Req, _Role) -> + ok. + +format_app_extend(App) -> + maps:remove(role, App). + +valid_role(?ROLE_API_DEFAULT) -> + ok; +valid_role(_) -> + {error, <<"Role does not exist">>}. + +-endif. + +maybe_migrate_table(Fields) -> + case mnesia:table_info(?APP, attributes) =:= Fields of + true -> + ok; + false -> + TransFun = fun(App) -> + case App of + {?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt} -> + #?APP{ + name = Name, + api_key = Key, + api_secret_hash = Hash, + enable = Enable, + desc = Desc, + expired_at = ExpiredAt, + created_at = CreatedAt, + role = ?ROLE_API_VIEWER, + extra = #{} + }; + #?APP{} -> + App + end + end, + {atomic, ok} = mnesia:transform_table(?APP, TransFun, Fields, ?APP), + ok + end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 2a78f76fc..b04fbf270 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -41,37 +41,42 @@ t_bootstrap_file(_) -> File = "./bootstrap_api_keys.txt", ok = file:write_file(File, Bin), update_file(File), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-1">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-1">>)), %% relaunch to check if the table is changed. Bin1 = <<"test-1:new-secret-1\ntest-2:new-secret-2">>, ok = file:write_file(File, Bin1), update_file(File), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), %% not error when bootstrap_file is empty update_file(<<>>), update_file("./bootstrap_apps_not_exist.txt"), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-1">>)), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-2">>)), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-1">>, <<"secret-1">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-2">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"new-secret-1">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-2">>, <<"new-secret-2">>)), %% bad format BadBin = <<"test-1:secret-11\ntest-2 secret-12">>, ok = file:write_file(File, BadBin), update_file(File), ?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()), - ?assertEqual(ok, emqx_mgmt_auth:authorize(TestPath, <<"test-1">>, <<"secret-11">>)), - ?assertMatch({error, _}, emqx_mgmt_auth:authorize(TestPath, <<"test-2">>, <<"secret-12">>)), + ?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-11">>)), + ?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-12">>)), update_file(<<>>), ok. +auth_authorize(Path, Key, Secret) -> + FakePath = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri("/fake")), + FakeReq = #{method => <<"GET">>, path => FakePath}, + emqx_mgmt_auth:authorize(Path, FakeReq, Key, Secret). + update_file(File) -> ?assertMatch({ok, _}, emqx:update_config([<<"api_key">>], #{<<"bootstrap_file">> => File})). diff --git a/rel/i18n/emqx_mgmt_api_api_keys.hocon b/rel/i18n/emqx_mgmt_api_api_keys.hocon index 85d5c4ec7..8acbe60d0 100644 --- a/rel/i18n/emqx_mgmt_api_api_keys.hocon +++ b/rel/i18n/emqx_mgmt_api_api_keys.hocon @@ -30,4 +30,7 @@ format.desc: format.label: """Unique and format by [a-zA-Z0-9-_]""" +role.desc: +"""Role for this API""" + } From db3915d4721459b81ea72ca7382eaaf223e161bc Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 17 Oct 2023 13:44:37 +0800 Subject: [PATCH 002/111] fix(rbac): update changes && fix CI errors --- apps/emqx/test/emqx_common_test_http.erl | 4 +++- apps/emqx_dashboard/src/emqx_dashboard.erl | 2 +- changes/ee/feat-11766.en.md | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changes/ee/feat-11766.en.md diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 1034e310f..83cf02019 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -17,7 +17,6 @@ -module(emqx_common_test_http). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl"). -export([ request_api/3, @@ -34,6 +33,9 @@ -define(DEFAULT_APP_ID, <<"default_appid">>). -define(DEFAULT_APP_SECRET, <<"default_app_secret">>). +%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl +-define(ROLE_API_SUPERUSER, <<"api_administrator">>). + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index fbd801410..96ff3e167 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -212,7 +212,7 @@ listener_name(Protocol) -> -if(?EMQX_RELEASE_EDITION =/= ee). %% dialyzer complains about the `unauthorized_role' clause... --dialyzer({no_match, [authorize/1]}). +-dialyzer({no_match, [authorize/1, api_key_authorize/3]}). -endif. authorize(Req) -> diff --git a/changes/ee/feat-11766.en.md b/changes/ee/feat-11766.en.md new file mode 100644 index 000000000..80925c907 --- /dev/null +++ b/changes/ee/feat-11766.en.md @@ -0,0 +1,8 @@ +Implemented a preliminary Role-Based Access Control for the REST API. + + In this version, there are three predefined roles: + - Administrator: This role could access all resources. + + - Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API. + + - Publisher: This role is special for MQTT messages publish, it can only access publish-related endpoints. From 26ec860d966b9f4f2bc5461d76f3694549fa0072 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 17 Oct 2023 14:38:16 +0800 Subject: [PATCH 003/111] feat(backup): add migration mechanism when import backup data --- apps/emqx/src/bhvrs/emqx_db_backup.erl | 17 +++ apps/emqx_management/src/emqx_mgmt_auth.erl | 72 +++++++---- .../src/emqx_mgmt_data_backup.erl | 116 +++++++++++++++--- .../test/emqx_mgmt_data_backup_SUITE.erl | 16 ++- 4 files changed, 180 insertions(+), 41 deletions(-) diff --git a/apps/emqx/src/bhvrs/emqx_db_backup.erl b/apps/emqx/src/bhvrs/emqx_db_backup.erl index fddbdb1d0..95a142c0e 100644 --- a/apps/emqx/src/bhvrs/emqx_db_backup.erl +++ b/apps/emqx/src/bhvrs/emqx_db_backup.erl @@ -16,4 +16,21 @@ -module(emqx_db_backup). +-type traverse_break_reason() :: over | migrate. + -callback backup_tables() -> [mria:table()]. + +%% validate the backup +%% return `ok` to traverse the next item +%% return `{ok, over}` to finish the traverse +%% return `{ok, migrate}` to call the migration callback +-callback validate_mnesia_backup(tuple()) -> + ok + | {ok, traverse_break_reason()} + | {error, term()}. + +-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}. + +-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]). + +-export_type([traverse_break_reason/0]). diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index c39e3888d..71e0ad11b 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -38,7 +38,7 @@ -export([authorize/4]). -export([post_config_update/5]). --export([backup_tables/0]). +-export([backup_tables/0, validate_mnesia_backup/1, migrate_mnesia_backup/1]). %% Internal exports (RPC) -export([ @@ -84,6 +84,35 @@ mnesia(boot) -> backup_tables() -> [?APP]. +validate_mnesia_backup({schema, _Tab, CreateList} = Schema) -> + case emqx_mgmt_data_backup:default_validate_mnesia_backup(Schema) of + ok -> + {ok, over}; + _ -> + case proplists:get_value(attributes, CreateList) of + [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] -> + {ok, migrate}; + Fields -> + {error, {unknow_fields, Fields}} + end + end; +validate_mnesia_backup(_Other) -> + ok. + +migrate_mnesia_backup({schema, Tab, CreateList}) -> + case proplists:get_value(attributes, CreateList) of + [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] = Fields -> + NewFields = Fields ++ [role, extra], + CreateList2 = lists:keyreplace( + attributes, 1, CreateList, {attributes, NewFields} + ), + {ok, {schema, Tab, CreateList2}}; + Fields -> + {error, {unknow_fields, Fields}} + end; +migrate_mnesia_backup(Data) -> + {ok, do_table_migrate(Data)}. + post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) -> #{bootstrap_file := File} = NewConf, case init_bootstrap_file(File) of @@ -203,7 +232,9 @@ ensure_not_undefined(New, _Old) -> New. to_map(Apps) when is_list(Apps) -> [to_map(App) || App <- Apps]; -to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D}) -> +to_map(#?APP{ + name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D, role = Role +}) -> #{ name => N, api_key => K, @@ -211,7 +242,8 @@ to_map(#?APP{name = N, api_key = K, enable = E, expired_at = ET, created_at = CT expired_at => ET, created_at => CT, desc => D, - expired => is_expired(ET) + expired => is_expired(ET), + role => Role }. is_expired(undefined) -> false; @@ -397,24 +429,22 @@ maybe_migrate_table(Fields) -> true -> ok; false -> - TransFun = fun(App) -> - case App of - {?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt} -> - #?APP{ - name = Name, - api_key = Key, - api_secret_hash = Hash, - enable = Enable, - desc = Desc, - expired_at = ExpiredAt, - created_at = CreatedAt, - role = ?ROLE_API_VIEWER, - extra = #{} - }; - #?APP{} -> - App - end - end, + TransFun = fun do_table_migrate/1, {atomic, ok} = mnesia:transform_table(?APP, TransFun, Fields, ?APP), ok end. + +do_table_migrate({?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt}) -> + #?APP{ + name = Name, + api_key = Key, + api_secret_hash = Hash, + enable = Enable, + desc = Desc, + expired_at = ExpiredAt, + created_at = CreatedAt, + role = ?ROLE_API_DEFAULT, + extra = #{} + }; +do_table_migrate(#?APP{} = App) -> + App. diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 0717e8285..d60e5612f 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -24,6 +24,8 @@ format_error/1 ]). +-export([default_validate_mnesia_backup/1]). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -223,7 +225,15 @@ export_cluster_hocon(TarDescriptor, BackupBaseName, Opts) -> export_mnesia_tabs(TarDescriptor, BackupName, BackupBaseName, Opts) -> maybe_print("Exporting built-in database...~n", [], Opts), lists:foreach( - fun(Tab) -> export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts) end, + fun(Mod) -> + Tabs = Mod:backup_tables(), + lists:foreach( + fun(Tab) -> + export_mnesia_tab(TarDescriptor, Tab, BackupName, BackupBaseName, Opts) + end, + Tabs + ) + end, tabs_to_backup() ). @@ -259,7 +269,7 @@ tabs_to_backup() -> -endif. mnesia_tabs_to_backup() -> - lists:flatten([M:backup_tables() || M <- find_behaviours(emqx_db_backup)]). + lists:flatten([M || M <- find_behaviours(emqx_db_backup)]). mnesia_backup_name(Path, TabName) -> filename:join([Path, ?BACKUP_MNESIA_DIR, atom_to_list(TabName)]). @@ -364,36 +374,42 @@ import_mnesia_tabs(BackupDir, Opts) -> maybe_print("Importing built-in database...~n", [], Opts), filter_errors( lists:foldr( - fun(Tab, Acc) -> Acc#{Tab => import_mnesia_tab(BackupDir, Tab, Opts)} end, + fun(Mod, Acc) -> + Tabs = Mod:backup_tables(), + lists:foldr( + fun(Tab, InAcc) -> + InAcc#{Tab => import_mnesia_tab(BackupDir, Mod, Tab, Opts)} + end, + Acc, + Tabs + ) + end, #{}, tabs_to_backup() ) ). -import_mnesia_tab(BackupDir, TabName, Opts) -> +import_mnesia_tab(BackupDir, Mod, TabName, Opts) -> MnesiaBackupFileName = mnesia_backup_name(BackupDir, TabName), case filelib:is_regular(MnesiaBackupFileName) of true -> maybe_print("Importing ~p database table...~n", [TabName], Opts), - restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts); + restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts); false -> maybe_print("No backup file for ~p database table...~n", [TabName], Opts), ?SLOG(info, #{msg => "missing_mnesia_backup", table => TabName, backup => BackupDir}), ok end. -restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) -> - Validated = - catch mnesia:traverse_backup( - MnesiaBackupFileName, mnesia_backup, dummy, read_only, fun validate_mnesia_backup/2, 0 - ), +restore_mnesia_tab(BackupDir, MnesiaBackupFileName, Mod, TabName, Opts) -> + Validated = validate_mnesia_backup(MnesiaBackupFileName, Mod), try case Validated of - {ok, _} -> + {ok, #{backup_file := BackupFile}} -> %% As we use keep_tables option, we don't need to modify 'copies' (nodes) %% in a backup file before restoring it, as `mnsia:restore/2` will ignore %% backed-up schema and keep the current table schema unchanged - Restored = mnesia:restore(MnesiaBackupFileName, [{default_op, keep_tables}]), + Restored = mnesia:restore(BackupFile, [{default_op, keep_tables}]), case Restored of {atomic, [TabName]} -> ok; @@ -425,17 +441,81 @@ restore_mnesia_tab(BackupDir, MnesiaBackupFileName, TabName, Opts) -> %% NOTE: if backup file is valid, we keep traversing it, though we only need to validate schema. %% Looks like there is no clean way to abort traversal without triggering any error reporting, %% `mnesia_bup:read_schema/2` is an option but its direct usage should also be avoided... -validate_mnesia_backup({schema, Tab, CreateList} = Schema, Acc) -> +validate_mnesia_backup(MnesiaBackupFileName, Mod) -> + Init = #{backup_file => MnesiaBackupFileName}, + Validated = + catch mnesia:traverse_backup( + MnesiaBackupFileName, + mnesia_backup, + dummy, + read_only, + mnesia_backup_validator(Mod), + Init + ), + case Validated of + ok -> + {ok, Init}; + {error, {_, over}} -> + {ok, Init}; + {error, {_, migrate}} -> + migrate_mnesia_backup(MnesiaBackupFileName, Mod, Init); + Error -> + Error + end. + +%% if the module has validator callback, use it else use the default +mnesia_backup_validator(Mod) -> + Validator = + case erlang:function_exported(Mod, validate_mnesia_backup, 1) of + true -> + fun Mod:validate_mnesia_backup/1; + _ -> + fun default_validate_mnesia_backup/1 + end, + fun(Schema, Acc) -> + case Validator(Schema) of + ok -> + {[Schema], Acc}; + {ok, Break} -> + throw({error, Break}); + Error -> + throw(Error) + end + end. + +default_validate_mnesia_backup({schema, Tab, CreateList}) -> ImportAttributes = proplists:get_value(attributes, CreateList), Attributes = mnesia:table_info(Tab, attributes), - case ImportAttributes =/= Attributes of + case ImportAttributes == Attributes of true -> - throw({error, different_table_schema}); + ok; false -> - {[Schema], Acc} + {error, different_table_schema} end; -validate_mnesia_backup(Other, Acc) -> - {[Other], Acc}. +default_validate_mnesia_backup(_Other) -> + ok. + +migrate_mnesia_backup(MnesiaBackupFileName, Mod, Acc) -> + case erlang:function_exported(Mod, migrate_mnesia_backup, 1) of + true -> + MigrateFile = MnesiaBackupFileName ++ ".migrate", + Migrator = fun(Schema, InAcc) -> + case Mod:migrate_mnesia_backup(Schema) of + {ok, NewSchema} -> + {[NewSchema], InAcc}; + Error -> + throw(Error) + end + end, + catch mnesia:traverse_backup( + MnesiaBackupFileName, + MigrateFile, + Migrator, + Acc#{backup_file := MigrateFile} + ); + _ -> + {error, no_migrator} + end. extract_backup(BackupFileName) -> BackupDir = root_backup_dir(), diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index 46566bd6f..c98ccf676 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(ROLE_SUPERUSER, <<"administrator">>). +-define(ROLE_API_SUPERUSER, <<"api_administrator">>). -define(BOOTSTRAP_BACKUP, "emqx-export-test-bootstrap-ce.tar.gz"). all() -> @@ -56,7 +57,7 @@ init_per_testcase(TC = t_verify_imported_mnesia_tab_on_cluster, Config) -> [{cluster, cluster(TC, Config)} | setup(TC, Config)]; init_per_testcase(t_mnesia_bad_tab_schema, Config) -> meck:new(emqx_mgmt_data_backup, [passthrough]), - meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [data_backup_test]), + meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [?MODULE]), setup(TC, Config); init_per_testcase(TC, Config) -> setup(TC, Config). @@ -99,7 +100,15 @@ t_cluster_hocon_export_import(Config) -> ?assertEqual(Exp, emqx_mgmt_data_backup:import(FileName)), ?assertEqual(RawConfAfterImport, emqx:get_raw_config([])), %% lookup file inside /backup - ?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))). + ?assertEqual(Exp, emqx_mgmt_data_backup:import(filename:basename(FileName))), + + %% backup data migration test + ?assertMatch([_, _, _], ets:tab2list(emqx_app)), + ?assertMatch( + {ok, #{name := <<"key_to_export2">>, role := ?ROLE_API_SUPERUSER}}, + emqx_mgmt_auth:read(<<"key_to_export2">>) + ), + ok. t_ee_to_ce_backup(Config) -> case emqx_release:edition() of @@ -329,6 +338,9 @@ t_verify_imported_mnesia_tab_on_cluster(Config) -> timer:sleep(3000), ?assertEqual(AllUsers, lists:sort(rpc:call(ReplicantNode, mnesia, dirty_all_keys, [Tab]))). +backup_tables() -> + [data_backup_test]. + t_mnesia_bad_tab_schema(_Config) -> OldAttributes = [id, name, description], ok = create_test_tab(OldAttributes), From 467a2bc9b140fa00b4f4c89389fbeb5443ff4a0b Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 17 Oct 2023 16:12:18 +0800 Subject: [PATCH 004/111] test(rbac): add test cases for RBAC in REST API --- apps/emqx_management/src/emqx_mgmt_auth.erl | 9 +- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 115 +++++++++++++++++- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 71e0ad11b..03a411b45 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -42,7 +42,7 @@ %% Internal exports (RPC) -export([ - do_update/4, + do_update/5, do_delete/1, do_create_app/3, do_force_create_app/3 @@ -149,12 +149,12 @@ read(Name) -> update(Name, Enable, ExpiredAt, Desc, Role) -> case valid_role(Role) of ok -> - trans(fun ?MODULE:do_update/4, [Name, Enable, ExpiredAt, Desc]); + trans(fun ?MODULE:do_update/5, [Name, Enable, ExpiredAt, Desc, Role]); Error -> Error end. -do_update(Name, Enable, ExpiredAt, Desc) -> +do_update(Name, Enable, ExpiredAt, Desc, Role) -> case mnesia:read(?APP, Name, write) of [] -> mnesia:abort(not_found); @@ -163,7 +163,8 @@ do_update(Name, Enable, ExpiredAt, Desc) -> App0#?APP{ expired_at = ExpiredAt, enable = ensure_not_undefined(Enable, Enable0), - desc = ensure_not_undefined(Desc, Desc0) + desc = ensure_not_undefined(Desc, Desc0), + role = Role }, ok = mnesia:write(App), to_map(App) diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index b04fbf270..8243b18ff 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -19,12 +19,26 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_dashboard/include/emqx_dashboard_rbac.hrl"). + +-if(?EMQX_RELEASE_EDITION == ee). +-define(EE_CASES, [ + t_ee_create, + t_ee_update, + t_ee_authorize_viewer, + t_ee_authorize_admin, + t_ee_authorize_publisher +]). +-else. +-define(EE_CASES, []). +-endif. all() -> [{group, parallel}, {group, sequence}]. suite() -> [{timetrap, {minutes, 1}}]. groups() -> [ {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]}, + {parallel, [parallel], ?EE_CASES}, {sequence, [], [t_bootstrap_file, t_create_failed]} ]. @@ -222,6 +236,102 @@ t_create_unexpired_app(_Config) -> ?assertMatch(#{<<"expired_at">> := <<"infinity">>}, Create2), ok. +t_ee_create(_Config) -> + Name = <<"EMQX-EE-API-KEY-1">>, + {ok, Create} = create_app(Name, #{role => ?ROLE_API_VIEWER}), + ?assertMatch( + #{ + <<"api_key">> := _, + <<"api_secret">> := _, + <<"created_at">> := _, + <<"desc">> := _, + <<"enable">> := true, + <<"expired_at">> := _, + <<"name">> := Name, + <<"role">> := ?ROLE_API_VIEWER + }, + Create + ), + + {ok, App} = read_app(Name), + ?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_VIEWER}, App). + +t_ee_update(_Config) -> + Name = <<"EMQX-EE-API-UPDATE-KEY">>, + {ok, _} = create_app(Name, #{role => ?ROLE_API_VIEWER}), + + Change = #{ + desc => <<"NoteVersion1"/utf8>>, + enable => false, + role => ?ROLE_API_SUPERUSER + }, + {ok, Update1} = update_app(Name, Change), + ?assertEqual(?ROLE_API_SUPERUSER, maps:get(<<"role">>, Update1)), + + {ok, App} = read_app(Name), + ?assertMatch(#{<<"name">> := Name, <<"role">> := ?ROLE_API_SUPERUSER}, App). + +t_ee_authorize_viewer(_Config) -> + Name = <<"EMQX-EE-API-AUTHORIZE-KEY-VIEWER">>, + {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{ + role => ?ROLE_API_VIEWER + }), + BasicHeader = emqx_common_test_http:auth_header( + binary_to_list(ApiKey), + binary_to_list(ApiSecret) + ), + + BanPath = emqx_mgmt_api_test_util:api_path(["banned"]), + ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)), + ?assertMatch( + {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader) + ). + +t_ee_authorize_admin(_Config) -> + Name = <<"EMQX-EE-API-AUTHORIZE-KEY-ADMIN">>, + {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{ + role => ?ROLE_API_SUPERUSER + }), + BasicHeader = emqx_common_test_http:auth_header( + binary_to_list(ApiKey), + binary_to_list(ApiSecret) + ), + + BanPath = emqx_mgmt_api_test_util:api_path(["banned"]), + ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader)), + ?assertMatch( + {ok, _}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader) + ). + +t_ee_authorize_publisher(_Config) -> + Name = <<"EMQX-EE-API-AUTHORIZE-KEY-PUBLISHER">>, + {ok, #{<<"api_key">> := ApiKey, <<"api_secret">> := ApiSecret}} = create_app(Name, #{ + role => ?ROLE_API_PUBLISHER + }), + BasicHeader = emqx_common_test_http:auth_header( + binary_to_list(ApiKey), + binary_to_list(ApiSecret) + ), + + BanPath = emqx_mgmt_api_test_util:api_path(["banned"]), + Publish = emqx_mgmt_api_test_util:api_path(["publish"]), + ?assertMatch( + {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(get, BanPath, BasicHeader) + ), + ?assertMatch( + {error, {_, 403, _}}, emqx_mgmt_api_test_util:request_api(delete, BanPath, BasicHeader) + ), + ?_assertMatch( + {ok, _}, + emqx_mgmt_api_test_util:request_api( + post, + Publish, + [], + BasicHeader, + #{topic => <<"t/t_ee_authorize_publisher">>, payload => <<"hello">>} + ) + ). + list_app() -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), @@ -239,10 +349,13 @@ read_app(Name) -> end. create_app(Name) -> + create_app(Name, #{}). + +create_app(Name, Extra) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), ExpiredAt = to_rfc3339(erlang:system_time(second) + 1000), - App = #{ + App = Extra#{ name => Name, expired_at => ExpiredAt, desc => <<"Note"/utf8>>, From b2948666fb90ece9f11ab1bbf868267b1144ca4f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 17 Oct 2023 23:45:59 +0800 Subject: [PATCH 005/111] chore: add saml sso redirect login log --- apps/emqx_dashboard/src/emqx_dashboard_admin.erl | 7 +++++++ .../emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index e9aac164b..438c9c246 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -207,8 +207,15 @@ add_user_(Username, Password, Role, Desc) -> description = Desc }, mnesia:write(Admin), + ?SLOG(info, #{msg => "dashboard_sso_user_added", username => Username, role => Role}), flatten_username(#{username => Username, role => Role, description => Desc}); [_] -> + ?SLOG(info, #{ + msg => "dashboard_sso_user_add_failed", + reason => "username_already_exists", + username => Username, + role => Role + }), mnesia:abort(<<"username_already_exist">>) end. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl index ccc40e2c6..949938884 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl @@ -95,6 +95,10 @@ sp_saml_callback(post, Req) -> State = #{enable := true} -> case (provider(saml)):callback(Req, State) of {redirect, Redirect} -> + ?SLOG(info, #{ + msg => "dashboard_saml_sso_login_successful", + redirect => "SAML login successful. Redirecting with LoginMeta." + }), Redirect; {error, Reason} -> ?SLOG(info, #{ From 1f8478623f788650884f284b43faed735242b7ba Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 17 Oct 2023 23:46:33 +0800 Subject: [PATCH 006/111] chore: bump esaml to support Entra ID && bump app.src --- apps/emqx_dashboard_sso/rebar.config | 2 +- apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard_sso/rebar.config b/apps/emqx_dashboard_sso/rebar.config index 2691afbc1..874aca800 100644 --- a/apps/emqx_dashboard_sso/rebar.config +++ b/apps/emqx_dashboard_sso/rebar.config @@ -4,5 +4,5 @@ {deps, [ {emqx_ldap, {path, "../../apps/emqx_ldap"}}, {emqx_dashboard, {path, "../../apps/emqx_dashboard"}}, - {esaml, {git, "https://github.com/emqx/esaml", {tag, "v1.1.2"}}} + {esaml, {git, "https://github.com/emqx/esaml", {tag, "v1.1.3"}}} ]}. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src index e00a3cbfa..71788947b 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_sso, [ {description, "EMQX Dashboard Single Sign-On"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, [emqx_dashboard_sso_sup]}, {applications, [ kernel, From 8b5ec2a4d5d873130e966323ed7349e569a4b086 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 20 Oct 2023 11:46:31 +0800 Subject: [PATCH 007/111] fix: xref failed --- apps/emqx_management/src/emqx_mgmt_auth.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 03a411b45..216fc636b 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -204,7 +204,7 @@ authorize(_Path, Req, ApiKey, ApiSecret) -> case find_by_api_key(ApiKey) of {ok, true, ExpiredAt, SecretHash, Role} when ExpiredAt >= Now -> case emqx_dashboard_admin:verify_hash(ApiSecret, SecretHash) of - ok -> check_rbac(Req, Role); + ok -> check_rbac(Req, ApiKey, Role); error -> {error, "secret_error"} end; {ok, true, _ExpiredAt, _SecretHash, _Role} -> @@ -396,8 +396,8 @@ add_bootstrap_file(File, Dev, MP, Line) -> end. -if(?EMQX_RELEASE_EDITION == ee). -check_rbac(Req, Role) -> - case emqx_dashboard_rbac:check_rbac(Req, Role) of +check_rbac(Req, ApiKey, Role) -> + case emqx_dashboard_rbac:check_rbac(Req, ApiKey, Role) of true -> ok; _ -> @@ -412,7 +412,7 @@ valid_role(Role) -> -else. -check_rbac(_Req, _Role) -> +check_rbac(_Req, _ApiKey, _Role) -> ok. format_app_extend(App) -> From 926c804314c60e364542dbf68a740ef204fa5e0d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 17 Oct 2023 09:56:19 +0800 Subject: [PATCH 008/111] feat: add /audit http api to filter audit log --- apps/emqx/include/logger.hrl | 17 +- apps/emqx/src/config/emqx_config_logger.erl | 9 + apps/emqx_audit/README.md | 5 + apps/emqx_audit/include/emqx_audit.hrl | 39 ++ apps/emqx_audit/rebar.config | 2 + apps/emqx_audit/src/emqx_audit.app.src | 10 + apps/emqx_audit/src/emqx_audit.erl | 202 +++++++++ apps/emqx_audit/src/emqx_audit_api.erl | 397 ++++++++++++++++++ apps/emqx_audit/src/emqx_audit_app.erl | 27 ++ apps/emqx_audit/src/emqx_audit_sup.erl | 45 ++ apps/emqx_conf/src/emqx_conf_cli.erl | 15 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 6 +- .../src/emqx_dashboard_audit.erl | 101 ++++- .../src/emqx_gateway_api_clients.erl | 2 +- apps/emqx_machine/priv/reboot_lists.eterm | 3 +- apps/emqx_machine/src/emqx_machine_boot.erl | 5 +- .../src/emqx_machine_terminator.erl | 6 +- .../src/emqx_restricted_shell.erl | 6 +- apps/emqx_management/src/emqx_mgmt_api.erl | 4 +- mix.exs | 3 +- rel/i18n/emqx_audit_api.hocon | 58 +++ 21 files changed, 909 insertions(+), 53 deletions(-) create mode 100644 apps/emqx_audit/README.md create mode 100644 apps/emqx_audit/include/emqx_audit.hrl create mode 100644 apps/emqx_audit/rebar.config create mode 100644 apps/emqx_audit/src/emqx_audit.app.src create mode 100644 apps/emqx_audit/src/emqx_audit.erl create mode 100644 apps/emqx_audit/src/emqx_audit_api.erl create mode 100644 apps/emqx_audit/src/emqx_audit_app.erl create mode 100644 apps/emqx_audit/src/emqx_audit_sup.erl create mode 100644 rel/i18n/emqx_audit_api.hocon diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index d803f67be..58ebbbf1f 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -61,25 +61,28 @@ ) end). --define(AUDIT(_Level_, _From_, _Meta_), begin +-define(AUDIT(_LevelFun_, _MetaFun_), begin case emqx_config:get([log, audit], #{enable => false}) of #{enable := false} -> ok; #{enable := true, level := _AllowLevel_} -> + _Level_ = _LevelFun_, case logger:compare_levels(_AllowLevel_, _Level_) of _R_ when _R_ == lt; _R_ == eq -> - emqx_trace:log( - _Level_, - [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], - _Msg = undefined, - _Meta_#{from => _From_} - ); + ?LOG_AUDIT_EVENT(_Level_, _MetaFun_); gt -> ok end end end). +-define(LOG_AUDIT_EVENT(Level, M), begin + M1 = (M)#{time => logger:timestamp(), level => Level}, + Filter = [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], + emqx_trace:log(Level, Filter, undefined, M1), + emqx_audit:log(M1) +end). + %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index c675edb52..57502bba8 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -23,6 +23,8 @@ -export([post_config_update/5]). -export([filter_audit/2]). +-include("logger.hrl"). + -define(LOG, [log]). -define(AUDIT_HANDLER, emqx_audit). @@ -96,6 +98,7 @@ update_log_handlers(NewHandlers) -> ok. update_log_handler({removed, Id}) -> + audit("audit_disabled", Id), log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]), logger:remove_handler(Id); update_log_handler({Action, {handler, Id, Mod, Conf}}) -> @@ -104,6 +107,7 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> _ = logger:remove_handler(Id), case logger:add_handler(Id, Mod, Conf) of ok -> + audit("audit_enabled", Id), ok; %% Don't crash here, otherwise the cluster rpc will retry the wrong handler forever. {error, Reason} -> @@ -114,6 +118,11 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. +audit(Event, ?AUDIT_HANDLER) -> + ?LOG_AUDIT_EVENT(alert, #{event => Event, from => event}); +audit(_, _) -> + ok. + id_for_log(console) -> "log.console"; id_for_log(Other) -> "log.file." ++ atom_to_list(Other). diff --git a/apps/emqx_audit/README.md b/apps/emqx_audit/README.md new file mode 100644 index 000000000..48c625ed5 --- /dev/null +++ b/apps/emqx_audit/README.md @@ -0,0 +1,5 @@ +emqx_audit +===== + +Audit log for EMQX, empowers users to efficiently access the desired audit trail data +and facilitates auditing, compliance, troubleshooting, and security analysis. diff --git a/apps/emqx_audit/include/emqx_audit.hrl b/apps/emqx_audit/include/emqx_audit.hrl new file mode 100644 index 000000000..59536def9 --- /dev/null +++ b/apps/emqx_audit/include/emqx_audit.hrl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(AUDIT, emqx_audit). + +-record(?AUDIT, { + seq, + %% basic info + created_at, + node, + from, + source, + source_ip, + %% operation info + operation_id, + operation_type, + args, + operation_result, + failure, + %% request detail + http_method, + http_request, + http_status_code, + duration_ms, + extra +}). diff --git a/apps/emqx_audit/rebar.config b/apps/emqx_audit/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_audit/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_audit/src/emqx_audit.app.src b/apps/emqx_audit/src/emqx_audit.app.src new file mode 100644 index 000000000..96cdd11ce --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit.app.src @@ -0,0 +1,10 @@ +{application, emqx_audit, [ + {description, "Audit log for EMQX"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_audit_app, []}}, + {applications, [kernel, stdlib, emqx]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl new file mode 100644 index 000000000..4b96f00b8 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -0,0 +1,202 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_audit). + +%% API +-export([]). + +-behaviour(gen_server). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include("emqx_audit.hrl"). + +%% API +-export([start_link/1]). +-export([log/1]). + +%% gen_server callbacks +-export([ + init/1, + handle_continue/2, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]). +-define(CLEAN_EXPIRED_MS, 60 * 1000). + +to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) -> + #?AUDIT{ + created_at = erlang:system_time(microsecond), + node = node(), + operation_id = <<"">>, + operation_type = atom_to_binary(Cmd), + args = Args, + operation_result = <<"">>, + failure = <<"">>, + duration_ms = DurationMs, + from = cli, + source = <<"">>, + source_ip = <<"">>, + http_status_code = <<"">>, + http_method = <<"">>, + http_request = <<"">> + }; +to_audit(#{http_method := get}) -> + ok; +to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api -> + #{ + source := Source, + source_ip := SourceIp, + %% operation info + operation_id := OperationId, + operation_type := OperationType, + operation_result := OperationResult, + %% request detail + http_status_code := StatusCode, + http_method := Method, + http_request := Request, + duration_ms := DurationMs + } = Log, + #?AUDIT{ + created_at = erlang:system_time(microsecond), + node = node(), + from = From, + source = Source, + source_ip = SourceIp, + %% operation info + operation_id = OperationId, + operation_type = OperationType, + operation_result = OperationResult, + failure = maps:get(failure, Log, <<"">>), + %% request detail + http_status_code = StatusCode, + http_method = Method, + http_request = Request, + duration_ms = DurationMs, + args = <<"">> + }; +to_audit(#{from := event, event := Event}) -> + #?AUDIT{ + created_at = erlang:system_time(microsecond), + node = node(), + from = event, + source = <<"">>, + source_ip = <<"">>, + %% operation info + operation_id = iolist_to_binary(Event), + operation_type = <<"">>, + operation_result = <<"">>, + failure = <<"">>, + %% request detail + http_status_code = <<"">>, + http_method = <<"">>, + http_request = <<"">>, + duration_ms = 0, + args = <<"">> + }; +to_audit(#{from := erlang_console, function := F, args := Args}) -> + #?AUDIT{ + created_at = erlang:system_time(microsecond), + node = node(), + from = erlang_console, + source = <<"">>, + source_ip = <<"">>, + %% operation info + operation_id = <<"">>, + operation_type = <<"">>, + operation_result = <<"">>, + failure = <<"">>, + %% request detail + http_status_code = <<"">>, + http_method = <<"">>, + http_request = <<"">>, + duration_ms = 0, + args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args])) + }. + +log(Log) -> + gen_server:cast(?MODULE, {write, to_audit(Log)}). + +start_link(Config) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Config], []). + +init([Config]) -> + erlang:process_flag(trap_exit, true), + ok = mria:create_table(?AUDIT, [ + {type, ordered_set}, + {rlog_shard, ?COMMON_SHARD}, + {storage, disc_copies}, + {record_name, ?AUDIT}, + {attributes, record_info(fields, ?AUDIT)} + ]), + {ok, Config, {continue, setup}}. + +handle_continue(setup, #{max_size := MaxSize} = State) -> + ok = mria:wait_for_tables([?AUDIT]), + LatestId = latest_id(), + clean_expired(LatestId, MaxSize), + {noreply, State#{latest_id => LatestId}}. + +handle_call(_Request, _From, State = #{}) -> + {reply, ok, State}. + +handle_cast({write, Log}, State = #{latest_id := LatestId}) -> + NewSeq = LatestId + 1, + Audit = Log#?AUDIT{seq = NewSeq}, + mnesia:dirty_write(?AUDIT, Audit), + {noreply, State#{latest_id => NewSeq}, ?CLEAN_EXPIRED_MS}; +handle_cast(_Request, State = #{}) -> + {noreply, State}. + +handle_info(timeout, State = #{max_size := MaxSize, latest_id := LatestId}) -> + clean_expired(LatestId, MaxSize), + {noreply, State#{latest_id => latest_id()}, hibernate}; +handle_info(_Info, State = #{}) -> + {noreply, State}. + +terminate(_Reason, _State = #{}) -> + ok. + +code_change(_OldVsn, State = #{}, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +clean_expired(LatestId, MaxSize) -> + Min = LatestId - MaxSize, + %% MS = ets:fun2ms(fun(#?AUDIT{seq = Seq}) when Seq =< Min -> true end), + MS = [{#?AUDIT{seq = '$1', _ = '_'}, [{'=<', '$1', Min}], [true]}], + NumDeleted = mnesia:ets(fun ets:select_delete/2, [?AUDIT, MS]), + ?SLOG(debug, #{ + msg => "clean_audit_log", + latest_id => LatestId, + min => Min, + deleted_number => NumDeleted + }), + ok. + +latest_id() -> + case mnesia:dirty_last(?AUDIT) of + '$end_of_table' -> 0; + Seq -> Seq + end. diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl new file mode 100644 index 000000000..f69d5d909 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -0,0 +1,397 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_audit_api). + +-behaviour(minirest_api). + +%% API +-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]). +-export([audit/2]). +-export([qs2ms/2, format/1]). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include("emqx_audit.hrl"). + +-import(hoconsc, [mk/2, ref/2, array/1]). + +-define(TAGS, ["Audit"]). + +-define(AUDIT_QS_SCHEMA, [ + {<<"node">>, atom}, + {<<"from">>, atom}, + {<<"source">>, binary}, + {<<"source_ip">>, binary}, + {<<"operation_id">>, binary}, + {<<"operation_type">>, binary}, + {<<"operation_result">>, atom}, + {<<"http_status_code">>, integer}, + {<<"http_method">>, atom}, + {<<"gte_created_at">>, timestamp}, + {<<"lte_created_at">>, timestamp}, + {<<"gte_duration_ms">>, timestamp}, + {<<"lte_duration_ms">>, timestamp} +]). + +namespace() -> "audit". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + ["/audit"]. + +schema("/audit") -> + #{ + 'operationId' => audit, + get => #{ + tags => ?TAGS, + description => ?DESC(audit_get), + parameters => [ + {node, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"emqx@127.0.0.1">>, + desc => ?DESC(filter_node) + })}, + {from, + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console, event]), #{ + in => query, + required => false, + example => <<"dashboard">>, + desc => ?DESC(filter_from) + })}, + {source, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"admin">>, + desc => ?DESC(filter_source) + })}, + {source_ip, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"127.0.0.1">>, + desc => ?DESC(filter_source_ip) + })}, + {operation_id, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"/rules/{id}">>, + desc => ?DESC(filter_operation_id) + })}, + {operation_type, + ?HOCON(binary(), #{ + in => query, + example => <<"rules">>, + required => false, + desc => ?DESC(filter_operation_type) + })}, + {operation_result, + ?HOCON(?ENUM([success, failure]), #{ + in => query, + example => failure, + required => false, + desc => ?DESC(filter_operation_result) + })}, + {http_status_code, + ?HOCON(integer(), #{ + in => query, + example => 200, + required => false, + desc => ?DESC(filter_http_status_code) + })}, + {http_method, + ?HOCON(?ENUM([post, put, delete]), #{ + in => query, + example => post, + required => false, + desc => ?DESC(filter_http_method) + })}, + {gte_duration_ms, + ?HOCON(integer(), #{ + in => query, + example => 0, + required => false, + desc => ?DESC(filter_gte_duration_ms) + })}, + {lte_duration_ms, + ?HOCON(integer(), #{ + in => query, + example => 1000, + required => false, + desc => ?DESC(filter_lte_duration_ms) + })}, + {gte_created_at, + ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + in => query, + required => false, + example => <<"2023-10-15T00:00:00.820384+08:00">>, + desc => ?DESC(filter_gte_created_at) + })}, + {lte_created_at, + ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + in => query, + example => <<"2023-10-16T00:00:00.820384+08:00">>, + required => false, + desc => ?DESC(filter_lte_created_at) + })}, + ref(emqx_dashboard_swagger, page), + ref(emqx_dashboard_swagger, limit) + ], + summary => <<"List audit logs">>, + responses => #{ + 200 => + emqx_dashboard_swagger:schema_with_example( + array(?REF(audit_list)), + audit_log_list_example() + ) + } + } + }. + +fields(audit_list) -> + [ + {data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})}, + {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} + ]; +fields(audit) -> + [ + {created_at, + ?HOCON( + emqx_utils_calendar:epoch_millisecond(), + #{ + desc => "The time when the log is created" + } + )}, + {node, + ?HOCON(binary(), #{ + desc => "The node name to which the log is created" + })}, + {from, + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console, event]), #{ + desc => "The source type of the log" + })}, + {source, + ?HOCON(binary(), #{ + desc => "The source of the log" + })}, + {source_ip, + ?HOCON(binary(), #{ + desc => "The source ip of the log" + })}, + {operation_id, + ?HOCON(binary(), #{ + desc => "The operation id of the log" + })}, + {operation_type, + ?HOCON(binary(), #{ + desc => "The operation type of the log" + })}, + {operation_result, + ?HOCON(?ENUM([success, failure]), #{ + desc => "The operation result of the log" + })}, + {http_status_code, + ?HOCON(integer(), #{ + desc => "The http status code of the log" + })}, + {http_method, + ?HOCON(?ENUM([post, put, delete]), #{ + desc => "The http method of the log" + })}, + {duration_ms, + ?HOCON(integer(), #{ + desc => "The duration of the log" + })}, + {args, + ?HOCON(?ARRAY(binary()), #{ + desc => "The args of the log" + })}, + {failure, + ?HOCON(?ARRAY(binary()), #{ + desc => "The failure of the log" + })}, + {http_request, + ?HOCON(?REF(http_request), #{ + desc => "The http request of the log" + })} + ]; +fields(http_request) -> + [ + {bindings, ?HOCON(map(), #{})}, + {body, ?HOCON(map(), #{})}, + {headers, ?HOCON(map(), #{})}, + {method, ?HOCON(?ENUM([post, put, delete]), #{})} + ]. + +audit(get, #{query_string := QueryString}) -> + case + emqx_mgmt_api:node_query( + node(), + ?AUDIT, + QueryString, + ?AUDIT_QS_SCHEMA, + fun ?MODULE:qs2ms/2, + fun ?MODULE:format/1 + ) + of + {error, page_limit_invalid} -> + {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + {error, Node, Error} -> + Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), + {500, #{code => <<"NODE_DOWN">>, message => Message}}; + Result -> + {200, Result} + end. + +qs2ms(_Tab, {Qs, _}) -> + #{ + match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []), + fuzzy_fun => undefined + }. + +gen_match_spec([], Audit, Conn) -> + [{Audit, Conn, ['$_']}]; +gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn); +gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn); +gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn); +gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn); +gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn); +gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn); +gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn); +gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn); +gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn); +gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]); +gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [ + {'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn + ]); +gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]); +gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [ + {'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn + ]). + +format(Audit) -> + #?AUDIT{ + created_at = CreatedAt, + node = Node, + from = From, + source = Source, + source_ip = SourceIp, + operation_id = OperationId, + operation_type = OperationType, + operation_result = OperationResult, + http_status_code = HttpStatusCode, + http_method = HttpMethod, + duration_ms = DurationMs, + args = Args, + failure = Failure, + http_request = HttpRequest + } = Audit, + #{ + created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond), + node => Node, + from => From, + source => Source, + source_ip => SourceIp, + operation_id => OperationId, + operation_type => OperationType, + operation_result => OperationResult, + http_status_code => HttpStatusCode, + http_method => HttpMethod, + duration_ms => DurationMs, + args => Args, + failure => Failure, + http_request => HttpRequest + }. + +audit_log_list_example() -> + #{ + data => [api_example(), cli_example()], + meta => #{ + <<"count">> => 2, + <<"hasnext">> => false, + <<"limit">> => 50, + <<"page">> => 1 + } + }. + +api_example() -> + #{ + <<"args">> => "", + <<"created_at">> => "2023-10-17T10:41:20.383993+08:00", + <<"duration_ms">> => 0, + <<"failure">> => "", + <<"from">> => "dashboard", + <<"http_method">> => "post", + <<"http_request">> => #{ + <<"bindings">> => #{}, + <<"body">> => #{ + <<"password">> => "******", + <<"username">> => "admin" + }, + <<"headers">> => #{ + <<"accept">> => "*/*", + <<"authorization">> => "******", + <<"connection">> => "keep-alive", + <<"content-length">> => "45", + <<"content-type">> => "application/json" + }, + <<"method">> => "post" + }, + <<"http_status_code">> => 200, + <<"node">> => "emqx@127.0.0.1", + <<"operation_id">> => "/login", + <<"operation_result">> => "success", + <<"operation_type">> => "login", + <<"source">> => "admin", + <<"source_ip">> => "127.0.0.1" + }. + +cli_example() -> + #{ + <<"args">> => [<<"show">>, <<"log">>], + <<"created_at">> => "2023-10-17T10:45:13.100426+08:00", + <<"duration_ms">> => 7, + <<"failure">> => "", + <<"from">> => "cli", + <<"http_method">> => "", + <<"http_request">> => "", + <<"http_status_code">> => "", + <<"node">> => "emqx@127.0.0.1", + <<"operation_id">> => "", + <<"operation_result">> => "", + <<"operation_type">> => "conf", + <<"source">> => "", + <<"source_ip">> => "" + }. diff --git a/apps/emqx_audit/src/emqx_audit_app.erl b/apps/emqx_audit/src/emqx_audit_app.erl new file mode 100644 index 000000000..2c7b086d5 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_app.erl @@ -0,0 +1,27 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_audit_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_audit_sup:start_link(). + +stop(_State) -> + ok. diff --git a/apps/emqx_audit/src/emqx_audit_sup.erl b/apps/emqx_audit/src/emqx_audit_sup.erl new file mode 100644 index 000000000..460ba90d6 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_sup.erl @@ -0,0 +1,45 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_audit_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 10, + period => 10 + }, + ChildSpecs = [ + #{ + id => emqx_audit, + start => {emqx_audit, start_link, [#{max_size => 5000}]}, + type => worker, + restart => transient, + shutdown => 1000 + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index ddabdae95..b47d1f961 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -108,15 +108,14 @@ admins(_) -> emqx_ctl:usage(usage_sync()). audit(Level, From, Log) -> - Log1 = redact(Log#{time => logger:timestamp()}), - ?AUDIT(Level, From, Log1). + ?AUDIT(Level, redact(Log#{from => From})). -redact(Logs = #{cmd := admins, args := ["add", Username, _Password | Rest]}) -> - Logs#{args => ["add", Username, "******" | Rest]}; -redact(Logs = #{cmd := admins, args := ["passwd", Username, _Password]}) -> - Logs#{args => ["passwd", Username, "******"]}; -redact(Logs = #{cmd := license, args := ["update", _License]}) -> - Logs#{args => ["update", "******"]}; +redact(Logs = #{cmd := admins, args := [<<"add">>, Username, _Password | Rest]}) -> + Logs#{args => [<<"add">>, Username, <<"******">> | Rest]}; +redact(Logs = #{cmd := admins, args := [<<"passwd">>, Username, _Password]}) -> + Logs#{args => [<<"passwd">>, Username, <<"******">>]}; +redact(Logs = #{cmd := license, args := [<<"update">>, _License]}) -> + Logs#{args => [<<"update">>, "******"]}; redact(Logs) -> Logs. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 96ff3e167..6d6d3d596 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -72,7 +72,7 @@ start_listeners(Listeners) -> base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, - log => fun emqx_dashboard_audit:log/1, + log => fun emqx_dashboard_audit:log/2, security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], swagger_global_spec => GlobalSpec, dispatch => dispatch(), @@ -222,7 +222,7 @@ authorize(Req) -> {bearer, Token} -> case emqx_dashboard_admin:verify_token(Req, Token) of {ok, Username} -> - {ok, #{auth_type => jwt_token, username => Username}}; + {ok, #{auth_type => jwt_token, source => Username}}; {error, token_timeout} -> {401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>}; {error, not_found} -> @@ -253,7 +253,7 @@ api_key_authorize(Req, Key, Secret) -> Path = cowboy_req:path(Req), case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of ok -> - {ok, #{auth_type => api_key, api_key => Key}}; + {ok, #{auth_type => api_key, source => Key}}; {error, <<"not_allowed">>} -> return_unauthorized( ?BAD_API_KEY_OR_SECRET, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index cb5c0f42b..4d3f6209e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -18,29 +18,84 @@ -include_lib("emqx/include/logger.hrl"). %% API --export([log/1]). +-export([log/2]). -log(Meta0) -> - #{req_start := ReqStart, req_end := ReqEnd, code := Code, method := Method} = Meta0, - Duration = erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond), - Level = level(Method, Code, Duration), - Username = maps:get(username, Meta0, <<"">>), - From = from(maps:get(auth_type, Meta0, "")), - Meta1 = maps:without([req_start, req_end], Meta0), - Meta2 = Meta1#{time => logger:timestamp(), duration_ms => Duration}, - Meta = emqx_utils:redact(Meta2), - ?AUDIT( - Level, - From, - Meta#{username => binary_to_list(Username), node => node()} - ), - ok. +%% todo filter high frequency events +-define(HIGH_FREQUENCY_EVENTS, [ + mqtt_subscribe, + mqtt_unsubscribe, + mqtt_subscribe_batch, + mqtt_unsubscribe_batch, + mqtt_publish, + mqtt_publish_batch, + kickout_client +]). -from(jwt_token) -> "dashboard"; -from(_) -> "rest_api". +log(#{code := Code, method := Method} = Meta, Req) -> + %% Keep level/2 and log_meta/1 inside of this ?AUDIT macro + ?AUDIT(level(Method, Code), log_meta(Meta, Req)). -level(get, _Code, _) -> debug; -level(_, Code, _) when Code >= 200 andalso Code < 300 -> info; -level(_, Code, _) when Code >= 300 andalso Code < 400 -> warning; -level(_, Code, _) when Code >= 400 andalso Code < 500 -> error; -level(_, _, _) -> critical. +log_meta(Meta, Req) -> + Meta1 = #{ + time => logger:timestamp(), + from => from(Meta), + source => source(Meta), + duration_ms => duration_ms(Meta), + source_ip => source_ip(Req), + operation_type => operation_type(Meta), + %% method for http filter api. + http_method => maps:get(method, Meta), + http_request => http_request(Meta), + http_status_code => maps:get(code, Meta), + operation_result => operation_result(Meta), + node => node() + }, + Meta2 = maps:without([req_start, req_end, method, headers, body, bindings, code], Meta), + emqx_utils:redact(maps:merge(Meta2, Meta1)). + +duration_ms(#{req_start := ReqStart, req_end := ReqEnd}) -> + erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond). + +from(Meta) -> + case maps:find(auth_type, Meta) of + {ok, jwt_token} -> + dashboard; + {ok, api_key} -> + rest_api; + error -> + case maps:find(operation_id, Meta) of + %% login api create jwt_token, so we don have authorization in it's headers + {ok, <<"/login">>} -> dashboard; + _ -> unknown + end + end. +source(#{source := Source}) -> Source; +source(#{operation_id := <<"/login">>, body := #{<<"username">> := Username}}) -> Username; +source(_Meta) -> <<"">>. + +source_ip(Req) -> + case cowboy_req:header(<<"x-forwarded-for">>, Req, undefined) of + undefined -> + {RemoteIP, _} = cowboy_req:peer(Req), + iolist_to_binary(inet:ntoa(RemoteIP)); + Addresses -> + hd(binary:split(Addresses, <<",">>)) + end. + +operation_type(Meta) -> + case maps:find(operation_id, Meta) of + {ok, OperationId} -> lists:nth(2, binary:split(OperationId, <<"/">>)); + _ -> <<"unknown">> + end. + +http_request(Meta) -> + maps:with([method, headers, bindings, body], Meta). + +operation_result(#{failure := _}) -> failure; +operation_result(_) -> success. + +level(get, _Code) -> debug; +level(_, Code) when Code >= 200 andalso Code < 300 -> info; +level(_, Code) when Code >= 300 andalso Code < 400 -> warning; +level(_, Code) when Code >= 400 andalso Code < 500 -> error; +level(_, _) -> critical. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index b698446b9..380ccfa6d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -33,7 +33,7 @@ ] ). -%% minirest/dashbaord_swagger behaviour callbacks +%% minirest/dashboard_swagger behaviour callbacks -export([ api_spec/0, paths/0, diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 16f901d27..5ea6eee70 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -124,7 +124,8 @@ emqx_ft, emqx_gcp_device, emqx_dashboard_rbac, - emqx_dashboard_sso + emqx_dashboard_sso, + emqx_audit ], %% must always be of type `load' ce_business_apps => diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 0d847376e..aa6180e23 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -47,7 +47,10 @@ post_boot() -> ok = ensure_apps_started(), ok = print_vsn(), ok = start_autocluster(), - ?AUDIT(alert, cli, #{time => logger:timestamp(), event => "emqx_start"}), + ?AUDIT(alert, #{ + event => "emqx_start", + from => event + }), ignore. -ifdef(TEST). diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index 4757507b5..d43d5fea9 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -67,9 +67,9 @@ graceful() -> %% @doc Shutdown the Erlang VM and wait indefinitely. graceful_wait() -> - ?AUDIT(alert, cli, #{ - time => logger:timestamp(), - event => emqx_gracefully_stop + ?AUDIT(alert, #{ + event => "emqx_gracefully_stop", + from => event }), ok = graceful(), exit_loop(). diff --git a/apps/emqx_machine/src/emqx_restricted_shell.erl b/apps/emqx_machine/src/emqx_restricted_shell.erl index a582a3cb8..4ddc913c0 100644 --- a/apps/emqx_machine/src/emqx_restricted_shell.erl +++ b/apps/emqx_machine/src/emqx_restricted_shell.erl @@ -112,11 +112,11 @@ max_heap_size_warning(MF, Args) -> log(_, {?MODULE, prompt_func}, [[{history, _}]]) -> ok; log(IsAllow, MF, Args) -> - ?AUDIT(warning, erlang_console, #{ - time => logger:timestamp(), + ?AUDIT(warning, #{ function => MF, args => pp_args(Args), - permission => IsAllow + permission => IsAllow, + from => erlang_console }), to_console(IsAllow, MF, Args). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index 0acffbe4a..dffa39ae5 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -341,11 +341,11 @@ do_select( try case maps:get(continuation, QueryState, undefined) of undefined -> - ets:select(Tab, Ms, Limit); + ets:select_reverse(Tab, Ms, Limit); Continuation -> %% XXX: Repair is necessary because we pass Continuation back %% and forth through the nodes in the `do_cluster_query` - ets:select(ets:repair_continuation(Continuation, Ms)) + ets:select_reverse(ets:repair_continuation(Continuation, Ms)) end catch exit:_ = Exit -> diff --git a/mix.exs b/mix.exs index 3817b5121..3551951fd 100644 --- a/mix.exs +++ b/mix.exs @@ -214,7 +214,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_azure_event_hub, :emqx_gcp_device, :emqx_dashboard_rbac, - :emqx_dashboard_sso + :eqmx_dashboard_sso, + :emqx_audit ]) end diff --git a/rel/i18n/emqx_audit_api.hocon b/rel/i18n/emqx_audit_api.hocon new file mode 100644 index 000000000..89c335f12 --- /dev/null +++ b/rel/i18n/emqx_audit_api.hocon @@ -0,0 +1,58 @@ +emqx_audit_api { + +audit_get.desc: +"""Get audit logs based on filter API, empowers users to efficiently +access the desired audit trail data and facilitates auditing, compliance, +troubleshooting, and security analysis""" + +audit_get.label: +"List audit logs" + +filter_node.desc: +"Filter logs based on the node name to which the logs are created." + +filter_from.desc: +""""Filter logs based on source type, valid values include: +`dashboard`: Dashboard request logs, requiring the use of a jwt_token. +`rest_api`: API KEY request logs. +`cli`: The emqx command line logs. +`erlang_console`: The emqx remote_console run function logs. +`event`: Logs related to events such as emqx_start, emqx_stop, audit_enabled, and audit_disabled.""" + +filter_source.desc: +""""Filter logs based on source, Possible values are: +The login username when logs are generated from the dashboard. +The API Keys when logs are generated from the REST API. +empty string when logs are generated from CLI, Erlang console, or an event.""" + +filter_source_ip.desc: +"Filter logs based on source ip when logs are generated from dashboard and REST API." + +filter_operation_id.desc: +"Filter log with swagger's operation_id when logs are generated from dashboard and REST API." + +filter_operation_type.desc: +"Filter logs with operation type." + +filter_operation_result.desc: +"Filter logs with operation result." + +filter_http_status_code.desc: +"Filter The HTTP API with response code when logs are generated from dashboard and REST API." + +filter_http_method.desc: +"Filter The HTTP API Request with method when logs are generated from dashboard and REST API." + +filter_gte_duration_ms.desc: +"Filter logs with a duration greater than or equal to given microseconds." + +filter_lte_duration_ms.desc: +"Filter logs with a duration less than or equal to given microseconds." + +filter_gte_created_at.desc: +"Filter logs with a creation time greater than or equal to the given timestamp, rfc3339 or timestamp(millisecond)" + +filter_lte_created_at.desc: +"Filter logs with a creation time less than or equal to the given timestamp, rfc3339 or timestamp(millisecond)" + +} From 141061c1e2e45dd3558599430824124b5fc8ed23 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 17 Oct 2023 22:14:47 +0800 Subject: [PATCH 009/111] feat: add max_filter_size and ignore_high_frequency_request config --- apps/emqx/include/logger.hrl | 9 +- apps/emqx/src/config/emqx_config_logger.erl | 3 +- apps/emqx_audit/src/emqx_audit.erl | 73 +++++--- apps/emqx_audit/src/emqx_audit_sup.erl | 2 +- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 170 ++++++++++++++++++ .../src/emqx_dashboard_audit.erl | 68 ++++--- .../src/emqx_enterprise.app.src | 2 +- .../src/emqx_enterprise_schema.erl | 18 ++ mix.exs | 2 +- rebar.config | 2 +- rel/i18n/emqx_conf_schema.hocon | 13 ++ 11 files changed, 302 insertions(+), 60 deletions(-) create mode 100644 apps/emqx_audit/test/emqx_audit_api_SUITE.erl diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 58ebbbf1f..67f125e5f 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -69,20 +69,13 @@ end). _Level_ = _LevelFun_, case logger:compare_levels(_AllowLevel_, _Level_) of _R_ when _R_ == lt; _R_ == eq -> - ?LOG_AUDIT_EVENT(_Level_, _MetaFun_); + emqx_audit:log(_Level_, _MetaFun_); gt -> ok end end end). --define(LOG_AUDIT_EVENT(Level, M), begin - M1 = (M)#{time => logger:timestamp(), level => Level}, - Filter = [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], - emqx_trace:log(Level, Filter, undefined, M1), - emqx_audit:log(M1) -end). - %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 57502bba8..89e439a2a 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -118,8 +118,9 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. +-dialyzer({nowarn_function, [audit/2]}). audit(Event, ?AUDIT_HANDLER) -> - ?LOG_AUDIT_EVENT(alert, #{event => Event, from => event}); + emqx_audit:log(alert, #{event => Event, from => event}); audit(_, _) -> ok. diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index 4b96f00b8..64e76ef9b 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -25,8 +25,8 @@ -include("emqx_audit.hrl"). %% API --export([start_link/1]). --export([log/1]). +-export([start_link/0]). +-export([log/1, log/2]). %% gen_server callbacks -export([ @@ -132,14 +132,21 @@ to_audit(#{from := erlang_console, function := F, args := Args}) -> args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args])) }. +log(_Level, undefined) -> + ok; +log(Level, Meta1) -> + Meta2 = Meta1#{time => logger:timestamp(), level => Level}, + Filter = [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], + emqx_trace:log(Level, Filter, undefined, Meta2), + emqx_audit:log(Meta2). + log(Log) -> gen_server:cast(?MODULE, {write, to_audit(Log)}). -start_link(Config) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Config], []). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). -init([Config]) -> - erlang:process_flag(trap_exit, true), +init([]) -> ok = mria:create_table(?AUDIT, [ {type, ordered_set}, {rlog_shard, ?COMMON_SHARD}, @@ -147,28 +154,25 @@ init([Config]) -> {record_name, ?AUDIT}, {attributes, record_info(fields, ?AUDIT)} ]), - {ok, Config, {continue, setup}}. + {ok, #{}, {continue, setup}}. -handle_continue(setup, #{max_size := MaxSize} = State) -> +handle_continue(setup, #{} = State) -> ok = mria:wait_for_tables([?AUDIT]), - LatestId = latest_id(), - clean_expired(LatestId, MaxSize), - {noreply, State#{latest_id => LatestId}}. + clean_expired(), + {noreply, State}. handle_call(_Request, _From, State = #{}) -> {reply, ok, State}. -handle_cast({write, Log}, State = #{latest_id := LatestId}) -> - NewSeq = LatestId + 1, - Audit = Log#?AUDIT{seq = NewSeq}, - mnesia:dirty_write(?AUDIT, Audit), - {noreply, State#{latest_id => NewSeq}, ?CLEAN_EXPIRED_MS}; +handle_cast({write, Log}, State) -> + _ = write_log(Log), + {noreply, State#{}, ?CLEAN_EXPIRED_MS}; handle_cast(_Request, State = #{}) -> {noreply, State}. -handle_info(timeout, State = #{max_size := MaxSize, latest_id := LatestId}) -> - clean_expired(LatestId, MaxSize), - {noreply, State#{latest_id => latest_id()}, hibernate}; +handle_info(timeout, State = #{}) -> + clean_expired(), + {noreply, State, hibernate}; handle_info(_Info, State = #{}) -> {noreply, State}. @@ -182,7 +186,33 @@ code_change(_OldVsn, State = #{}, _Extra) -> %%% Internal functions %%%=================================================================== -clean_expired(LatestId, MaxSize) -> +write_log(Log) -> + case + mria:transaction( + ?COMMON_SHARD, + fun(L) -> + New = + case mnesia:last(?AUDIT) of + '$end_of_table' -> 1; + LastId -> LastId + 1 + end, + mnesia:write(L#?AUDIT{seq = New}) + end, + [Log] + ) + of + {atomic, ok} -> + ok; + Reason -> + ?SLOG(warning, #{ + msg => "write_audit_log_failed", + reason => Reason + }) + end. + +clean_expired() -> + MaxSize = max_size(), + LatestId = latest_id(), Min = LatestId - MaxSize, %% MS = ets:fun2ms(fun(#?AUDIT{seq = Seq}) when Seq =< Min -> true end), MS = [{#?AUDIT{seq = '$1', _ = '_'}, [{'=<', '$1', Min}], [true]}], @@ -200,3 +230,6 @@ latest_id() -> '$end_of_table' -> 0; Seq -> Seq end. + +max_size() -> + emqx_conf:get([log, audit, max_filter_size], 5000). diff --git a/apps/emqx_audit/src/emqx_audit_sup.erl b/apps/emqx_audit/src/emqx_audit_sup.erl index 460ba90d6..0671a9e0f 100644 --- a/apps/emqx_audit/src/emqx_audit_sup.erl +++ b/apps/emqx_audit/src/emqx_audit_sup.erl @@ -36,7 +36,7 @@ init([]) -> ChildSpecs = [ #{ id => emqx_audit, - start => {emqx_audit, start_link, [#{max_size => 5000}]}, + start => {emqx_audit, start_link, []}, type => worker, restart => transient, shutdown => 1000 diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl new file mode 100644 index 000000000..6fb860b6e --- /dev/null +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -0,0 +1,170 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_audit_api_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-define(CONF_DEFAULT, #{ + node => + #{ + name => "emqx1@127.0.0.1", + cookie => "emqxsecretcookie", + data_dir => "data" + }, + log => #{ + audit => + #{ + enable => true, + ignore_high_frequency_request => true, + level => info, + max_filter_size => 15, + rotation_count => 2, + rotation_size => "10MB", + time_offset => "system" + } + } +}). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + emqx_config:erase_all(), + emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]), + ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), + application:set_env(emqx, boot_modules, []), + emqx_conf_cli:load(), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]). + +t_http_api(_) -> + process_flag(trap_exit, true), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones), + ?assertMatch(#{<<"max_qos_allowed">> := 1}, Res), + {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + ?assertMatch( + #{ + <<"data">> := [ + #{ + <<"from">> := <<"rest_api">>, + <<"operation_id">> := <<"/configs/global_zone">>, + <<"source_ip">> := <<"127.0.0.1">>, + <<"source">> := _, + <<"http_request">> := #{ + <<"method">> := <<"put">>, + <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}}, + <<"bindings">> := _, + <<"headers">> := #{<<"authorization">> := <<"******">>} + }, + <<"http_status_code">> := 200, + <<"operation_result">> := <<"success">>, + <<"operation_type">> := <<"configs">> + } + ] + }, + emqx_utils_json:decode(Res1, [return_maps]) + ), + ok. + +t_cli(_Config) -> + ok = emqx_ctl:run_command(["conf", "show", "log"]), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]), + ?assertMatch( + [ + #{ + <<"from">> := <<"cli">>, + <<"operation_id">> := <<"">>, + <<"source_ip">> := <<"">>, + <<"operation_type">> := <<"conf">>, + <<"args">> := [<<"show">>, <<"log">>], + <<"node">> := _, + <<"source">> := <<"">>, + <<"http_request">> := <<"">> + } + ], + Data + ), + + %% check filter + {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader), + #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]), + ?assertEqual(Data, Data1), + {ok, Res2} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "from=erlang_console", AuthHeader + ), + ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])), + ok. + +t_kickout_clients_without_log(_) -> + process_flag(trap_exit, true), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + {ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath), + kickout_clients(), + {ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath), + ?assertEqual(AuditLogs1, AuditLogs2), + ok. + +kickout_clients() -> + ClientId1 = <<"client1">>, + ClientId2 = <<"client2">>, + ClientId3 = <<"client3">>, + + {ok, C1} = emqtt:start_link(#{ + clientid => ClientId1, + proto_ver => v5, + properties => #{'Session-Expiry-Interval' => 120} + }), + {ok, _} = emqtt:connect(C1), + {ok, C2} = emqtt:start_link(#{clientid => ClientId2}), + {ok, _} = emqtt:connect(C2), + {ok, C3} = emqtt:start_link(#{clientid => ClientId3}), + {ok, _} = emqtt:connect(C3), + + timer:sleep(300), + + %% get /clients + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), + ClientsMeta = maps:get(<<"meta">>, ClientsResponse), + ClientsPage = maps:get(<<"page">>, ClientsMeta), + ClientsLimit = maps:get(<<"limit">>, ClientsMeta), + ClientsCount = maps:get(<<"count">>, ClientsMeta), + ?assertEqual(ClientsPage, 1), + ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()), + ?assertEqual(ClientsCount, 3), + + %% kickout clients + KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]), + KickoutBody = [ClientId1, ClientId2, ClientId3], + {ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody), + + {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]), + ?assertMatch(#{<<"meta">> := #{<<"count">> := 0}}, ClientsResponse2). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 4d3f6209e..704e849bc 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -20,15 +20,15 @@ %% API -export([log/2]). -%% todo filter high frequency events --define(HIGH_FREQUENCY_EVENTS, [ - mqtt_subscribe, - mqtt_unsubscribe, - mqtt_subscribe_batch, - mqtt_unsubscribe_batch, - mqtt_publish, - mqtt_publish_batch, - kickout_client +%% filter high frequency events +-define(HIGH_FREQUENCY_REQUESTS, [ + <<"/clients/:clientid/publish">>, + <<"/clients/:clientid/subscribe">>, + <<"/clients/:clientid/unsubscribe">>, + <<"/clients/:clientid/publish/bulk">>, + <<"/clients/:clientid/unsubscribe/bulk">>, + <<"/clients/:clientid/subscribe/bulk">>, + <<"/clients/kickout/bulk">> ]). log(#{code := Code, method := Method} = Meta, Req) -> @@ -36,22 +36,31 @@ log(#{code := Code, method := Method} = Meta, Req) -> ?AUDIT(level(Method, Code), log_meta(Meta, Req)). log_meta(Meta, Req) -> - Meta1 = #{ - time => logger:timestamp(), - from => from(Meta), - source => source(Meta), - duration_ms => duration_ms(Meta), - source_ip => source_ip(Req), - operation_type => operation_type(Meta), - %% method for http filter api. - http_method => maps:get(method, Meta), - http_request => http_request(Meta), - http_status_code => maps:get(code, Meta), - operation_result => operation_result(Meta), - node => node() - }, - Meta2 = maps:without([req_start, req_end, method, headers, body, bindings, code], Meta), - emqx_utils:redact(maps:merge(Meta2, Meta1)). + #{operation_id := OperationId} = Meta, + case + lists:member(OperationId, ?HIGH_FREQUENCY_REQUESTS) andalso + ignore_high_frequency_request() + of + true -> + undefined; + false -> + Meta1 = #{ + time => logger:timestamp(), + from => from(Meta), + source => source(Meta), + duration_ms => duration_ms(Meta), + source_ip => source_ip(Req), + operation_type => operation_type(Meta), + %% method for http filter api. + http_method => maps:get(method, Meta), + http_request => http_request(Meta), + http_status_code => maps:get(code, Meta), + operation_result => operation_result(Meta), + node => node() + }, + Meta2 = maps:without([req_start, req_end, method, headers, body, bindings, code], Meta), + emqx_utils:redact(maps:merge(Meta2, Meta1)) + end. duration_ms(#{req_start := ReqStart, req_end := ReqEnd}) -> erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond). @@ -84,8 +93,10 @@ source_ip(Req) -> operation_type(Meta) -> case maps:find(operation_id, Meta) of - {ok, OperationId} -> lists:nth(2, binary:split(OperationId, <<"/">>)); - _ -> <<"unknown">> + {ok, OperationId} -> + lists:nth(2, binary:split(OperationId, <<"/">>, [global])); + _ -> + <<"unknown">> end. http_request(Meta) -> @@ -99,3 +110,6 @@ level(_, Code) when Code >= 200 andalso Code < 300 -> info; level(_, Code) when Code >= 300 andalso Code < 400 -> warning; level(_, Code) when Code >= 400 andalso Code < 500 -> error; level(_, _) -> critical. + +ignore_high_frequency_request() -> + emqx_conf:get([log, audit, ignore_high_frequency_request], true). diff --git a/apps/emqx_enterprise/src/emqx_enterprise.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src index 37d31c5ec..1a5359db6 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ {application, emqx_enterprise, [ {description, "EMQX Enterprise Edition"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 0cf5850c8..5537c3259 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -78,6 +78,24 @@ fields("log_audit_handler") -> desc => ?DESC(emqx_conf_schema, "log_file_handler_max_size"), importance => ?IMPORTANCE_MEDIUM } + )}, + {"max_filter_size", + hoconsc:mk( + range(10, 30000), + #{ + default => 5000, + desc => ?DESC(emqx_conf_schema, "audit_log_max_filter_limit"), + importance => ?IMPORTANCE_MEDIUM + } + )}, + {"ignore_high_frequency_request", + hoconsc:mk( + boolean(), + #{ + default => true, + desc => ?DESC(emqx_conf_schema, "audit_log_ignore_high_frequency_request"), + importance => ?IMPORTANCE_MEDIUM + } )} ] ++ CommonConfs1; fields(Name) -> diff --git a/mix.exs b/mix.exs index 3551951fd..a088d89f7 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.0", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true}, - {:minirest, github: "emqx/minirest", tag: "1.3.13", override: true}, + {:minirest, github: "emqx/minirest", tag: "1.3.14", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, diff --git a/rebar.config b/rebar.config index e2e1a7cf0..9f66553cc 100644 --- a/rebar.config +++ b/rebar.config @@ -65,7 +65,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.13"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.14"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index fde0f7ff3..ff05eaf6a 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -725,6 +725,19 @@ audit_handler_level.desc: audit_handler_level.label: """Log Level""" +audit_log_max_filter_limit.desc: +"""Maximum size of the filter.""" + +audit_log_max_filter_limit.label: +"""Max Filter Limit""" + +audit_log_ignore_high_frequency_request.desc: +"""Ignore high frequency requests to avoid flooding the audit log. +such publish/subscribe kickout http api requests are ignored.""" + +audit_log_ignore_high_frequency_request.label: +"""Ignore High Frequency Request""" + desc_rpc.desc: """EMQX uses a library called gen_rpc for inter-broker communication.
Most of the time the default config should work, From befc4845440c982bbd7de609001301fce571c75f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 18 Oct 2023 08:02:02 +0800 Subject: [PATCH 010/111] chore: add changelog entry and fix xref warning --- apps/emqx/src/config/emqx_config_logger.erl | 6 +- apps/emqx_audit/BSL.txt | 94 +++++++++++++++++++++ changes/ee/feat-11773.en.md | 1 + mix.exs | 1 + rebar.config | 1 + 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_audit/BSL.txt create mode 100644 changes/ee/feat-11773.en.md diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 89e439a2a..449d5207d 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -118,11 +118,15 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. --dialyzer({nowarn_function, [audit/2]}). +-if(?EMQX_RELEASE_EDITION == ee). audit(Event, ?AUDIT_HANDLER) -> emqx_audit:log(alert, #{event => Event, from => event}); audit(_, _) -> ok. +-else. +audit(_, _) -> + ok. +-endif. id_for_log(console) -> "log.console"; id_for_log(Other) -> "log.file." ++ atom_to_list(Other). diff --git a/apps/emqx_audit/BSL.txt b/apps/emqx_audit/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_audit/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/changes/ee/feat-11773.en.md b/changes/ee/feat-11773.en.md new file mode 100644 index 000000000..fcc96b1e6 --- /dev/null +++ b/changes/ee/feat-11773.en.md @@ -0,0 +1 @@ +Support audit log filter via dashboard (http api). diff --git a/mix.exs b/mix.exs index a088d89f7..a978c3ced 100644 --- a/mix.exs +++ b/mix.exs @@ -330,6 +330,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_gateway_lwm2m, :emqx_gateway_exproto, :emqx_dashboard, + :emqx_dashboard_sso, :emqx_resource, :emqx_connector, :emqx_exhook, diff --git a/rebar.config b/rebar.config index 9f66553cc..75dc98556 100644 --- a/rebar.config +++ b/rebar.config @@ -102,6 +102,7 @@ {emqx_schema_parser,decode,3}, {emqx_schema_parser,encode,3}, {emqx_schema_registry,add_schema,1}, + {emqx_audit, log, 2}, emqx_exhook_pb, % generated code for protobuf emqx_exproto_pb % generated code for protobuf ]}. From 1d7aa9495a92397f35da76dd93b5e1ae8502e130 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 18 Oct 2023 09:10:41 +0800 Subject: [PATCH 011/111] chore: make spellcheck happy --- apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl | 2 ++ mix.exs | 2 +- rel/i18n/emqx_conf_schema.hocon | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl index e2aece927..bf1f358ea 100644 --- a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl @@ -95,6 +95,8 @@ t_audit_log_conf(_Config) -> <<"enable">> => false, <<"level">> => <<"info">>, <<"path">> => <<"log/audit.log">>, + <<"ignore_high_frequency_request">> => true, + <<"max_filter_size">> => 5000, <<"rotation_count">> => 10, <<"rotation_size">> => <<"50MB">>, <<"time_offset">> => <<"system">> diff --git a/mix.exs b/mix.exs index a978c3ced..f7ad79c3d 100644 --- a/mix.exs +++ b/mix.exs @@ -214,7 +214,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_azure_event_hub, :emqx_gcp_device, :emqx_dashboard_rbac, - :eqmx_dashboard_sso, + :emqx_dashboard_sso, :emqx_audit ]) end diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index ff05eaf6a..b68c44fcb 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -732,8 +732,8 @@ audit_log_max_filter_limit.label: """Max Filter Limit""" audit_log_ignore_high_frequency_request.desc: -"""Ignore high frequency requests to avoid flooding the audit log. -such publish/subscribe kickout http api requests are ignored.""" +"""Ignore high frequency requests to avoid flooding the audit log, +such as publish/subscribe kick out http api requests are ignored.""" audit_log_ignore_high_frequency_request.label: """Ignore High Frequency Request""" From 32c1f1cca6a3320613838086a662a98f74111b03 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 18 Oct 2023 14:39:43 +0800 Subject: [PATCH 012/111] chore: inlude emqx as emqx_audit's deps --- apps/emqx_audit/rebar.config | 5 ++++- apps/emqx_audit/src/emqx_audit.erl | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx_audit/rebar.config b/apps/emqx_audit/rebar.config index 2656fd554..fac0f9b07 100644 --- a/apps/emqx_audit/rebar.config +++ b/apps/emqx_audit/rebar.config @@ -1,2 +1,5 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index 64e76ef9b..debe0608b 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -16,12 +16,10 @@ -module(emqx_audit). -%% API --export([]). - -behaviour(gen_server). --include_lib("emqx/include/logger.hrl"). + -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). -include("emqx_audit.hrl"). %% API From f381961108e3cba2b398f3e50255a0f5d5609aa9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 18 Oct 2023 16:50:41 +0800 Subject: [PATCH 013/111] fix: macro EMQX_RELEASE_EDITION when `emqx` run as standalnoe app --- apps/emqx/src/config/emqx_config_logger.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 449d5207d..87baef627 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -118,6 +118,8 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. +-ifdef(EMQX_RELEASE_EDITION). + -if(?EMQX_RELEASE_EDITION == ee). audit(Event, ?AUDIT_HANDLER) -> emqx_audit:log(alert, #{event => Event, from => event}); @@ -128,6 +130,11 @@ audit(_, _) -> ok. -endif. +-else. +audit(_, _) -> + ok. +-endif. + id_for_log(console) -> "log.console"; id_for_log(Other) -> "log.file." ++ atom_to_list(Other). From c97fe796e31ef3c3863424ee8f9edc66a9d5957f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 20 Oct 2023 09:47:24 +0800 Subject: [PATCH 014/111] chore: update emqx_audit's license --- apps/emqx_audit/include/emqx_audit.hrl | 14 +------------- apps/emqx_audit/src/emqx_audit.erl | 14 +------------- apps/emqx_audit/src/emqx_audit_api.erl | 15 ++------------- apps/emqx_audit/src/emqx_audit_app.erl | 14 +------------- apps/emqx_audit/src/emqx_audit_sup.erl | 14 +------------- 5 files changed, 6 insertions(+), 65 deletions(-) diff --git a/apps/emqx_audit/include/emqx_audit.hrl b/apps/emqx_audit/include/emqx_audit.hrl index 59536def9..1b4349387 100644 --- a/apps/emqx_audit/include/emqx_audit.hrl +++ b/apps/emqx_audit/include/emqx_audit.hrl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -define(AUDIT, emqx_audit). diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index debe0608b..4477bbd8b 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_audit). diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl index f69d5d909..aaa364464 100644 --- a/apps/emqx_audit/src/emqx_audit_api.erl +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -1,18 +1,7 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- + -module(emqx_audit_api). -behaviour(minirest_api). diff --git a/apps/emqx_audit/src/emqx_audit_app.erl b/apps/emqx_audit/src/emqx_audit_app.erl index 2c7b086d5..aa8fa1a39 100644 --- a/apps/emqx_audit/src/emqx_audit_app.erl +++ b/apps/emqx_audit/src/emqx_audit_app.erl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_audit_app). diff --git a/apps/emqx_audit/src/emqx_audit_sup.erl b/apps/emqx_audit/src/emqx_audit_sup.erl index 0671a9e0f..b3a5ca985 100644 --- a/apps/emqx_audit/src/emqx_audit_sup.erl +++ b/apps/emqx_audit/src/emqx_audit_sup.erl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_audit_sup). From 6a8b2dc1f913f695746993595b6e535f344c1c00 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 20 Oct 2023 14:57:31 +0800 Subject: [PATCH 015/111] fix: bad high frequency request name --- apps/emqx_dashboard/src/emqx_dashboard_audit.erl | 4 ++-- mix.exs | 1 + rebar.config.erl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 704e849bc..78608ee9b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -22,10 +22,10 @@ %% filter high frequency events -define(HIGH_FREQUENCY_REQUESTS, [ - <<"/clients/:clientid/publish">>, + <<"/publish">>, <<"/clients/:clientid/subscribe">>, <<"/clients/:clientid/unsubscribe">>, - <<"/clients/:clientid/publish/bulk">>, + <<"/publish/bulk">>, <<"/clients/:clientid/unsubscribe/bulk">>, <<"/clients/:clientid/subscribe/bulk">>, <<"/clients/kickout/bulk">> diff --git a/mix.exs b/mix.exs index f7ad79c3d..ed869c414 100644 --- a/mix.exs +++ b/mix.exs @@ -331,6 +331,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_gateway_exproto, :emqx_dashboard, :emqx_dashboard_sso, + :emqx_audit, :emqx_resource, :emqx_connector, :emqx_exhook, diff --git a/rebar.config.erl b/rebar.config.erl index dd9bd1b04..d9277d7e8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -110,6 +110,7 @@ is_community_umbrella_app("apps/emqx_bridge_azure_event_hub") -> false; is_community_umbrella_app("apps/emqx_gcp_device") -> false; is_community_umbrella_app("apps/emqx_dashboard_rbac") -> false; is_community_umbrella_app("apps/emqx_dashboard_sso") -> false; +is_community_umbrella_app("apps/emqx_audit") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> From ef692596f7f4f8635ad61f6cd6ff5213deab167f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 20 Oct 2023 15:22:30 +0800 Subject: [PATCH 016/111] fix: don't crash when 401 and 403 unauthorize --- apps/emqx/include/http_api.hrl | 1 + apps/emqx_dashboard/src/emqx_dashboard.erl | 3 +-- .../src/emqx_dashboard_audit.erl | 26 ++++++++++--------- rel/i18n/emqx_audit_api.hocon | 2 +- rel/i18n/emqx_conf_schema.hocon | 2 +- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index ba1438374..0f6372584 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -17,6 +17,7 @@ %% HTTP API Auth -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET'). +-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>). %% Bad Request -define(BAD_REQUEST, 'BAD_REQUEST'). diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 6d6d3d596..cf4330e34 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -260,8 +260,7 @@ api_key_authorize(Req, Key, Secret) -> <<"Not allowed, Check api_key/api_secret">> ); {error, unauthorized_role} -> - {403, 'UNAUTHORIZED_ROLE', - <<"This API Key don't have permission to access this resource">>}; + {403, 'UNAUTHORIZED_ROLE', ?API_KEY_NOT_ALLOW_MSG}; {error, _} -> return_unauthorized( ?BAD_API_KEY_OR_SECRET, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index 78608ee9b..c2ef1a99f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -17,6 +17,7 @@ -module(emqx_dashboard_audit). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/http_api.hrl"). %% API -export([log/2]). @@ -65,19 +66,20 @@ log_meta(Meta, Req) -> duration_ms(#{req_start := ReqStart, req_end := ReqEnd}) -> erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond). -from(Meta) -> - case maps:find(auth_type, Meta) of - {ok, jwt_token} -> - dashboard; - {ok, api_key} -> - rest_api; - error -> - case maps:find(operation_id, Meta) of - %% login api create jwt_token, so we don have authorization in it's headers - {ok, <<"/login">>} -> dashboard; - _ -> unknown - end +from(#{auth_type := jwt_token}) -> + dashboard; +from(#{auth_type := api_key}) -> + rest_api; +from(#{operation_id := <<"/login">>}) -> + dashboard; +from(#{code := Code} = Meta) when Code =:= 401 orelse Code =:= 403 -> + case maps:find(failure, Meta) of + {ok, #{code := 'BAD_API_KEY_OR_SECRET'}} -> rest_api; + {ok, #{code := 'UNAUTHORIZED_ROLE', message := ?API_KEY_NOT_ALLOW_MSG}} -> rest_api; + %% 'TOKEN_TIME_OUT' 'BAD_TOKEN' is dashboard code. + _ -> dashboard end. + source(#{source := Source}) -> Source; source(#{operation_id := <<"/login">>, body := #{<<"username">> := Username}}) -> Username; source(_Meta) -> <<"">>. diff --git a/rel/i18n/emqx_audit_api.hocon b/rel/i18n/emqx_audit_api.hocon index 89c335f12..37080838b 100644 --- a/rel/i18n/emqx_audit_api.hocon +++ b/rel/i18n/emqx_audit_api.hocon @@ -17,7 +17,7 @@ filter_from.desc: `rest_api`: API KEY request logs. `cli`: The emqx command line logs. `erlang_console`: The emqx remote_console run function logs. -`event`: Logs related to events such as emqx_start, emqx_stop, audit_enabled, and audit_disabled.""" +`event`: Logs related to events such as emqx_start, emqx_gracefully_stop, audit_enabled, and audit_disabled.""" filter_source.desc: """"Filter logs based on source, Possible values are: diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index b68c44fcb..ff2c3109a 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -726,7 +726,7 @@ audit_handler_level.label: """Log Level""" audit_log_max_filter_limit.desc: -"""Maximum size of the filter.""" +"""Store the latest N log entries in a database for allow `/audit` HTTP API to filter and retrieval of log data.""" audit_log_max_filter_limit.label: """Max Filter Limit""" From 8ae69874ee8fc7cd2e58dfc10d4bafcb7d5c1faa Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 24 Oct 2023 09:33:10 +0800 Subject: [PATCH 017/111] chore: add missing change log for saml SSO Entra Id support --- changes/ee/feat-11778.en.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ee/feat-11778.en.md diff --git a/changes/ee/feat-11778.en.md b/changes/ee/feat-11778.en.md new file mode 100644 index 000000000..30a291945 --- /dev/null +++ b/changes/ee/feat-11778.en.md @@ -0,0 +1,6 @@ +Support Azure Entra Id for saml single sign on. + +## Hight Light known issue: +Signed SAML `AuthnRequest` is not fully supported yet by Azure Entra ID. +See also for more details: +[single-sign-on-saml-protocol#signature](https://learn.microsoft.com/en-us/azure/active-directory/develop/single-sign-on-saml-protocol#signature) From 1c2f9321d193974a514c20ca2a675e6c88a767b3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 24 Oct 2023 14:45:04 +0700 Subject: [PATCH 018/111] feat(emqx): add file-sourced generic secrets These secrets follow the same `emqx_secret` convention of 0-arity functions. Also provide a simple HOCON schema module for use in application schemas. --- apps/emqx/src/emqx_schema_secret.erl | 107 ++++++++++++++++++ apps/emqx/src/emqx_secret.erl | 27 ++++- apps/emqx/test/emqx_secret_tests.erl | 69 +++++++++++ .../src/emqx_dashboard_swagger.erl | 3 + 4 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 apps/emqx/src/emqx_schema_secret.erl create mode 100644 apps/emqx/test/emqx_secret_tests.erl diff --git a/apps/emqx/src/emqx_schema_secret.erl b/apps/emqx/src/emqx_schema_secret.erl new file mode 100644 index 000000000..aa2cbcc84 --- /dev/null +++ b/apps/emqx/src/emqx_schema_secret.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc HOCON schema that defines _secret_ concept. +-module(emqx_schema_secret). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([mk/1]). + +%% HOCON Schema API +-export([convert_secret/2]). + +%% Target of `emqx_secret:wrap/3` +-export([load/1]). + +%% @doc Secret value. +-type t() :: binary(). + +%% @doc Source of the secret value. +%% * "file://...": file path to a file containing secret value. +%% * other binaries: secret value itself. +-type source() :: iodata(). + +-type secret() :: binary() | function(). +-reflect_type([secret/0]). + +-define(SCHEMA, #{ + required => false, + format => <<"password">>, + sensitive => true, + converter => fun ?MODULE:convert_secret/2 +}). + +-dialyzer({nowarn_function, source/1}). + +%% + +-spec mk(#{atom() => _}) -> hocon_schema:field_schema(). +mk(Overrides = #{}) -> + hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)). + +convert_secret(undefined, #{}) -> + undefined; +convert_secret(Secret, #{make_serializable := true}) -> + unicode:characters_to_binary(source(Secret)); +convert_secret(Secret, #{}) when is_function(Secret, 0) -> + Secret; +convert_secret(Secret, #{}) when is_integer(Secret) -> + wrap(integer_to_binary(Secret)); +convert_secret(Secret, #{}) -> + try unicode:characters_to_binary(Secret) of + String when is_binary(String) -> + wrap(String); + {error, _, _} -> + throw(invalid_string) + catch + error:_ -> + throw(invalid_type) + end. + +-spec wrap(source()) -> emqx_secret:t(t()). +wrap(Source) -> + try + _Secret = load(Source), + emqx_secret:wrap(?MODULE, load, Source) + catch + error:Reason -> + % NOTE: This should be a term serializable as JSON value. + throw(emqx_utils:format(Reason)) + end. + +-spec source(emqx_secret:t(t())) -> source(). +source(Secret) when is_function(Secret) -> + emqx_secret:term(Secret); +source(Secret) -> + Secret. + +%% + +-spec load(source()) -> t(). +load(<<"file://", Filename/binary>>) -> + load_file(Filename); +load(Secret) -> + Secret. + +load_file(Filename) -> + case file:read_file(Filename) of + {ok, Secret} -> + string:trim(Secret, trailing, [$\n]); + {error, Reason} -> + error({inaccessible_secret_file, Reason}, [Filename]) + end. diff --git a/apps/emqx/src/emqx_secret.erl b/apps/emqx/src/emqx_secret.erl index 72c4f3c08..ad0194201 100644 --- a/apps/emqx/src/emqx_secret.erl +++ b/apps/emqx/src/emqx_secret.erl @@ -19,7 +19,7 @@ -module(emqx_secret). %% API: --export([wrap/1, unwrap/1]). +-export([wrap/1, wrap/3, unwrap/1, term/1]). -export_type([t/1]). @@ -29,13 +29,38 @@ %% API funcions %%================================================================================ +%% @doc Wrap a term in a secret closure. +%% This effectively hides the term from any term formatting / printing code. +-spec wrap(T) -> t(T). wrap(Term) -> fun() -> Term end. +%% @doc Wrap a function call over a term in a secret closure. +%% This is slightly more flexible form of `wrap/1` with the same basic purpose. +-spec wrap(module(), atom(), _Term) -> t(_). +wrap(Module, Function, Term) -> + fun() -> + apply(Module, Function, [Term]) + end. + +%% @doc Unwrap a secret closure, revealing the secret. +%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped. +-spec unwrap(t(T)) -> T. unwrap(Term) when is_function(Term, 0) -> %% Handle potentially nested funs unwrap(Term()); unwrap(Term) -> Term. + +%% @doc Inspect the term wrapped in a secret closure. +-spec term(t(_)) -> _Term. +term(Wrap) when is_function(Wrap, 0) -> + case erlang:fun_info(Wrap, module) of + {module, ?MODULE} -> + {env, Env} = erlang:fun_info(Wrap, env), + lists:last(Env); + _ -> + error(badarg, [Wrap]) + end. diff --git a/apps/emqx/test/emqx_secret_tests.erl b/apps/emqx/test/emqx_secret_tests.erl new file mode 100644 index 000000000..ab0866c10 --- /dev/null +++ b/apps/emqx/test/emqx_secret_tests.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_secret_tests). + +-export([ident/1]). + +-include_lib("eunit/include/eunit.hrl"). + +wrap_unwrap_test() -> + ?assertEqual( + 42, + emqx_secret:unwrap(emqx_secret:wrap(42)) + ). + +unwrap_immediate_test() -> + ?assertEqual( + 42, + emqx_secret:unwrap(42) + ). + +wrap_unwrap_external_test() -> + ?assertEqual( + ident({foo, bar}), + emqx_secret:unwrap(emqx_secret:wrap(?MODULE, ident, {foo, bar})) + ). + +wrap_unwrap_transform_test() -> + ?assertEqual( + <<"this_was_an_atom">>, + emqx_secret:unwrap(emqx_secret:wrap(erlang, atom_to_binary, this_was_an_atom)) + ). + +wrap_term_test() -> + ?assertEqual( + 42, + emqx_secret:term(emqx_secret:wrap(42)) + ). + +wrap_external_term_test() -> + ?assertEqual( + this_was_an_atom, + emqx_secret:term(emqx_secret:wrap(erlang, atom_to_binary, this_was_an_atom)) + ). + +external_fun_term_error_test() -> + Term = {foo, bar}, + ?assertError( + badarg, + emqx_secret:term(fun() -> Term end) + ). + +%% + +ident(X) -> + X. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 75e93fdd1..f1759fb2d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -908,6 +908,9 @@ typename_to_spec("port_number()", _Mod) -> range("1..65535"); typename_to_spec("secret_access_key()", _Mod) -> #{type => string, example => <<"TW8dPwmjpjJJuLW....">>}; +typename_to_spec("secret()", _Mod) -> + %% TODO: ideally, this should be dispatched to the module that defines this type + #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}; typename_to_spec(Name, Mod) -> try_convert_to_spec(Name, Mod, [ fun try_remote_module_type/2, From 52f4519eebfc2a4df8790ffeb211d8fe66bf3c21 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 24 Oct 2023 14:52:25 +0700 Subject: [PATCH 019/111] feat(mqttbridge): support file-sourced secrets as passwords --- .../src/emqx_bridge_mqtt_connector.erl | 10 +- .../src/emqx_bridge_mqtt_connector_schema.erl | 8 +- .../test/emqx_bridge_mqtt_SUITE.erl | 183 +++++++++++------- .../test/emqx_bridge_mqtt_SUITE_data/password | 1 + 4 files changed, 126 insertions(+), 76 deletions(-) create mode 100644 apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index eb81c4b6e..ff5f0c2f2 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -326,7 +326,7 @@ mk_client_opts( ], Config ), - Options#{ + mk_client_opt_password(Options#{ hosts => [HostPort], clientid => clientid(ResourceId, ClientScope, Config), connect_timeout => 30, @@ -334,7 +334,13 @@ mk_client_opts( force_ping => true, ssl => EnableSsl, ssl_opts => maps:to_list(maps:remove(enable, Ssl)) - }. + }). + +mk_client_opt_password(Options = #{password := Secret}) -> + %% TODO: Teach `emqtt` to accept 0-arity closures as passwords. + Options#{password := emqx_secret:unwrap(Secret)}; +mk_client_opt_password(Options) -> + Options. ms_to_s(Ms) -> erlang:ceil(Ms / 1000). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 1dc3ca5f8..eb298c5ff 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -99,13 +99,9 @@ fields("server_configs") -> } )}, {password, - mk( - binary(), + emqx_schema_secret:mk( #{ - format => <<"password">>, - sensitive => true, - desc => ?DESC("password"), - converter => fun emqx_schema:password_converter/2 + desc => ?DESC("password") } )}, {clean_start, diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index bc0f2450a..6b5cc86da 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -21,13 +21,15 @@ -import(emqx_dashboard_api_test_helpers, [request/4, uri/1]). -include("emqx/include/emqx.hrl"). +-include("emqx/include/emqx_hooks.hrl"). +-include("emqx/include/asserts.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% output functions -export([inspect/3]). --define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(TYPE_MQTT, <<"mqtt">>). -define(BRIDGE_NAME_INGRESS, <<"ingress_mqtt_bridge">>). -define(BRIDGE_NAME_EGRESS, <<"egress_mqtt_bridge">>). @@ -38,14 +40,18 @@ -define(EGRESS_REMOTE_TOPIC, "egress_remote_topic"). -define(EGRESS_LOCAL_TOPIC, "egress_local_topic"). --define(SERVER_CONF(Username), #{ +-define(SERVER_CONF, #{ + <<"type">> => ?TYPE_MQTT, <<"server">> => <<"127.0.0.1:1883">>, - <<"username">> => Username, - <<"password">> => <<"">>, <<"proto_ver">> => <<"v4">>, <<"ssl">> => #{<<"enable">> => false} }). +-define(SERVER_CONF(Username, Password), (?SERVER_CONF)#{ + <<"username">> => Username, + <<"password">> => Password +}). + -define(INGRESS_CONF, #{ <<"remote">> => #{ <<"topic">> => <>, @@ -129,43 +135,32 @@ suite() -> [{timetrap, {seconds, 30}}]. init_per_suite(Config) -> - _ = application:load(emqx_conf), - ok = emqx_common_test_helpers:start_apps( + Apps = emqx_cth_suite:start( [ - emqx_rule_engine, + emqx_conf, emqx_bridge, + emqx_rule_engine, emqx_bridge_mqtt, - emqx_dashboard + {emqx_dashboard, + "dashboard {" + "\n listeners.http { bind = 18083 }" + "\n default_username = connector_admin" + "\n default_password = public" + "\n }"} ], - fun set_special_configs/1 + #{work_dir => emqx_cth_suite:work_dir(Config)} ), - ok = emqx_common_test_helpers:load_config( - emqx_rule_engine_schema, - <<"rule_engine {rules {}}">> - ), - ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT), - Config. + [{suite_apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([ - emqx_dashboard, - emqx_bridge_mqtt, - emqx_bridge, - emqx_rule_engine - ]), - ok. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>); -set_special_configs(_) -> - ok. +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)). init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), ok = snabbkaffe:start_trace(), Config. end_per_testcase(_, _Config) -> + ok = unhook_authenticate(), clear_resources(), snabbkaffe:stop(), ok. @@ -187,14 +182,84 @@ clear_resources() -> %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ + +t_conf_bridge_authn_anonymous(_) -> + ok = hook_authenticate(), + {ok, 201, _Bridge} = request( + post, + uri(["bridges"]), + ?SERVER_CONF#{ + <<"name">> => <<"t_conf_bridge_anonymous">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := undefined, password := undefined}} + ). + +t_conf_bridge_authn_password(_) -> + Username1 = <<"user1">>, + Password1 = <<"from-here">>, + ok = hook_authenticate(), + {ok, 201, _Bridge1} = request( + post, + uri(["bridges"]), + ?SERVER_CONF(Username1, Password1)#{ + <<"name">> => <<"t_conf_bridge_authn_password">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := Username1, password := Password1}} + ). + +t_conf_bridge_authn_passfile(Config) -> + DataDir = ?config(data_dir, Config), + Username2 = <<"user2">>, + PasswordFilename = filename:join(DataDir, "password"), + Password2 = <<"from-there">>, + ok = hook_authenticate(), + {ok, 201, _Bridge2} = request( + post, + uri(["bridges"]), + ?SERVER_CONF(Username2, iolist_to_binary(["file://", PasswordFilename]))#{ + <<"name">> => <<"t_conf_bridge_authn_passfile">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := Username2, password := Password2}} + ), + {ok, 400, #{<<"message">> := Message}} = request_json( + post, + uri(["bridges"]), + ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ + <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + } + ), + ?assertMatch( + #{<<"reason">> := <<"{inaccessible_secret_file,enoent}">>}, + emqx_utils_json:decode(Message) + ). + +hook_authenticate() -> + emqx_hooks:add('client.authenticate', {?MODULE, authenticate, [self()]}, ?HP_HIGHEST). + +unhook_authenticate() -> + emqx_hooks:del('client.authenticate', {?MODULE, authenticate}). + +authenticate(Credential, _, TestRunnerPid) -> + _ = TestRunnerPid ! {authenticate, Credential}, + ignore. + +%%------------------------------------------------------------------------------ + t_mqtt_conn_bridge_ingress(_) -> - User1 = <<"user1">>, %% create an MQTT bridge, using POST {ok, 201, Bridge} = request( post, uri(["bridges"]), - ServerConf = ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ServerConf = ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } @@ -249,7 +314,6 @@ t_mqtt_conn_bridge_ingress(_) -> ok. t_mqtt_conn_bridge_ingress_full_context(_Config) -> - User1 = <<"user1">>, IngressConf = emqx_utils_maps:deep_merge( ?INGRESS_CONF, @@ -258,8 +322,7 @@ t_mqtt_conn_bridge_ingress_full_context(_Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => IngressConf } @@ -297,8 +360,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> Ns = lists:seq(1, 10), BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<>>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"ingress">> => #{ <<"pool_size">> => PoolSize, @@ -337,8 +399,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> t_mqtt_egress_bridge_ignores_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"egress">> => ?EGRESS_CONF, <<"clean_start">> => false @@ -366,8 +427,7 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, @@ -392,9 +452,8 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ok. t_mqtt_conn_bridge_ingress_no_payload_template(_) -> - User1 = <<"user1">>, BridgeIDIngress = create_bridge( - ?SERVER_CONF(User1)#{ + ?SERVER_CONF#{ <<"type">> => ?TYPE_MQTT, <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF_NO_PAYLOAD_TEMPLATE @@ -428,10 +487,8 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> t_mqtt_conn_bridge_egress(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -473,11 +530,8 @@ t_mqtt_conn_bridge_egress(_) -> t_mqtt_conn_bridge_egress_no_payload_template(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, - BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF_NO_PAYLOAD_TEMPLATE } @@ -520,11 +574,9 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> ok. t_egress_custom_clientid_prefix(_Config) -> - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ + ?SERVER_CONF#{ <<"clientid_prefix">> => <<"my-custom-prefix">>, - <<"type">> => ?TYPE_MQTT, <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -545,17 +597,14 @@ t_egress_custom_clientid_prefix(_Config) -> ok. t_mqtt_conn_bridge_ingress_and_egress(_) -> - User1 = <<"user1">>, BridgeIDIngress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } ), BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -627,8 +676,7 @@ t_mqtt_conn_bridge_ingress_and_egress(_) -> t_ingress_mqtt_bridge_with_rules(_) -> BridgeIDIngress = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } @@ -712,8 +760,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> t_egress_mqtt_bridge_with_rules(_) -> BridgeIDEgress = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -789,10 +836,8 @@ t_egress_mqtt_bridge_with_rules(_) -> t_mqtt_conn_bridge_egress_reconnect(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ @@ -897,10 +942,8 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> ok. t_mqtt_conn_bridge_egress_async_reconnect(_) -> - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ @@ -1018,5 +1061,9 @@ request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), emqx_utils_json:decode(BridgeMetrics). +request_json(Method, Url, Body) -> + {ok, Code, Response} = request(Method, Url, Body), + {ok, Code, emqx_utils_json:decode(Response)}. + request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password new file mode 100644 index 000000000..d68418fda --- /dev/null +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password @@ -0,0 +1 @@ +from-there From 7a8a5926ab91a8475d9b49c96df2ea904772f2f0 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 24 Oct 2023 19:01:12 +0800 Subject: [PATCH 020/111] fix(rbac): adjust the role names --- apps/emqx/test/emqx_common_test_http.erl | 2 +- apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl | 6 +++--- apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 83cf02019..67473363e 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -34,7 +34,7 @@ -define(DEFAULT_APP_SECRET, <<"default_app_secret">>). %% from emqx_dashboard/include/emqx_dashboard_rbac.hrl --define(ROLE_API_SUPERUSER, <<"api_administrator">>). +-define(ROLE_API_SUPERUSER, <<"administrator">>). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl index 8f49464a4..386ae8bea 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl @@ -25,9 +25,9 @@ -define(ROLE_SUPERUSER, <<"administrator">>). -define(ROLE_DEFAULT, ?ROLE_SUPERUSER). --define(ROLE_API_VIEWER, <<"api_viewer">>). --define(ROLE_API_SUPERUSER, <<"api_administrator">>). --define(ROLE_API_PUBLISHER, <<"api_publisher">>). +-define(ROLE_API_VIEWER, <<"viewer">>). +-define(ROLE_API_SUPERUSER, <<"administrator">>). +-define(ROLE_API_PUBLISHER, <<"publisher">>). -define(ROLE_API_DEFAULT, ?ROLE_API_SUPERUSER). -endif. diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index c98ccf676..c384b55e8 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -23,7 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(ROLE_SUPERUSER, <<"administrator">>). --define(ROLE_API_SUPERUSER, <<"api_administrator">>). +-define(ROLE_API_SUPERUSER, <<"administrator">>). -define(BOOTSTRAP_BACKUP, "emqx-export-test-bootstrap-ce.tar.gz"). all() -> From e175c213a19f0e48d2a8cdadf0bbfd4fe7add57c Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 24 Oct 2023 22:28:53 +0800 Subject: [PATCH 021/111] fix(rbac): for compatibility with old data schema, extend the existing field as extra --- .../src/emqx_dashboard_rbac.erl | 8 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 104 +++++------------- 2 files changed, 34 insertions(+), 78 deletions(-) diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index bde3be6e6..cd38540dd 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -59,12 +59,12 @@ valid_role(Type, Role) -> %% =================================================================== check_rbac(?ROLE_SUPERUSER, _, _, _) -> true; -check_rbac(?ROLE_API_SUPERUSER, _, _, _) -> - true; +%%check_rbac(?ROLE_API_SUPERUSER, _, _, _) -> +%% true; check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) -> true; -check_rbac(?ROLE_API_VIEWER, <<"GET">>, _, _) -> - true; +%%check_rbac(?ROLE_API_VIEWER, <<"GET">>, _, _) -> +%% true; check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>, _) -> true; check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>, _) -> diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 216fc636b..f0d92ece7 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -38,7 +38,7 @@ -export([authorize/4]). -export([post_config_update/5]). --export([backup_tables/0, validate_mnesia_backup/1, migrate_mnesia_backup/1]). +-export([backup_tables/0]). %% Internal exports (RPC) -export([ @@ -53,18 +53,17 @@ -endif. -define(APP, emqx_app). --type api_user_role() :: binary(). -record(?APP, { name = <<>> :: binary() | '_', api_key = <<>> :: binary() | '_', api_secret_hash = <<>> :: binary() | '_', enable = true :: boolean() | '_', - desc = <<>> :: binary() | '_', + %% Since v5.4.0 the `desc` has changed to `extra` + %% desc = <<>> :: binary() | '_', + extra = #{} :: binary() | map() | '_', expired_at = 0 :: integer() | undefined | infinity | '_', - created_at = 0 :: integer() | '_', - role = ?ROLE_DEFAULT :: api_user_role() | '_', - extra = #{} :: map() | '_' + created_at = 0 :: integer() | '_' }). mnesia(boot) -> @@ -75,8 +74,7 @@ mnesia(boot) -> {storage, disc_copies}, {record_name, ?APP}, {attributes, Fields} - ]), - maybe_migrate_table(Fields). + ]). %%-------------------------------------------------------------------- %% Data backup @@ -84,35 +82,6 @@ mnesia(boot) -> backup_tables() -> [?APP]. -validate_mnesia_backup({schema, _Tab, CreateList} = Schema) -> - case emqx_mgmt_data_backup:default_validate_mnesia_backup(Schema) of - ok -> - {ok, over}; - _ -> - case proplists:get_value(attributes, CreateList) of - [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] -> - {ok, migrate}; - Fields -> - {error, {unknow_fields, Fields}} - end - end; -validate_mnesia_backup(_Other) -> - ok. - -migrate_mnesia_backup({schema, Tab, CreateList}) -> - case proplists:get_value(attributes, CreateList) of - [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] = Fields -> - NewFields = Fields ++ [role, extra], - CreateList2 = lists:keyreplace( - attributes, 1, CreateList, {attributes, NewFields} - ), - {ok, {schema, Tab, CreateList2}}; - Fields -> - {error, {unknow_fields, Fields}} - end; -migrate_mnesia_backup(Data) -> - {ok, do_table_migrate(Data)}. - post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) -> #{bootstrap_file := File} = NewConf, case init_bootstrap_file(File) of @@ -158,13 +127,13 @@ do_update(Name, Enable, ExpiredAt, Desc, Role) -> case mnesia:read(?APP, Name, write) of [] -> mnesia:abort(not_found); - [App0 = #?APP{enable = Enable0, desc = Desc0}] -> + [App0 = #?APP{enable = Enable0, extra = Extra0}] -> + #{desc := Desc0} = Extra = normalize_extra(Extra0), App = App0#?APP{ expired_at = ExpiredAt, enable = ensure_not_undefined(Enable, Enable0), - desc = ensure_not_undefined(Desc, Desc0), - role = Role + extra = Extra#{desc := ensure_not_undefined(Desc, Desc0), role := Role} }, ok = mnesia:write(App), to_map(App) @@ -220,10 +189,10 @@ find_by_api_key(ApiKey) -> case mria:ro_transaction(?COMMON_SHARD, Fun) of {atomic, [ #?APP{ - api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt, role = Role + api_secret_hash = SecretHash, enable = Enable, expired_at = ExpiredAt, extra = Extra } ]} -> - {ok, Enable, ExpiredAt, SecretHash, Role}; + {ok, Enable, ExpiredAt, SecretHash, get_role(Extra)}; _ -> {error, "not_found"} end. @@ -234,15 +203,16 @@ ensure_not_undefined(New, _Old) -> New. to_map(Apps) when is_list(Apps) -> [to_map(App) || App <- Apps]; to_map(#?APP{ - name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, desc = D, role = Role + name = N, api_key = K, enable = E, expired_at = ET, created_at = CT, extra = Extra0 }) -> + #{role := Role, desc := Desc} = normalize_extra(Extra0), #{ name => N, api_key => K, enable => E, expired_at => ET, created_at => CT, - desc => D, + desc => Desc, expired => is_expired(ET), role => Role }. @@ -256,11 +226,10 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) -> name = Name, enable = Enable, expired_at = ExpiredAt, - desc = Desc, + extra = #{desc => Desc, role => Role}, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_utils:gen_id(16)), - role = Role + api_key = list_to_binary(emqx_utils:gen_id(16)) }, case create_app(App) of {ok, Res} -> @@ -269,7 +238,7 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc, Role) -> Error end. -create_app(App = #?APP{api_key = ApiKey, name = Name, role = Role}) -> +create_app(App = #?APP{api_key = ApiKey, name = Name, extra = #{role := Role}}) -> case valid_role(Role) of ok -> trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]); @@ -364,7 +333,7 @@ add_bootstrap_file(File, Dev, MP, Line) -> #?APP{ enable = true, expired_at = infinity, - desc = ?BOOTSTRAP_TAG, + extra = #{desc => ?BOOTSTRAP_TAG, role => ?ROLE_API_DEFAULT}, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), api_key = AppKey @@ -395,6 +364,18 @@ add_bootstrap_file(File, Dev, MP, Line) -> throw(#{file => File, line => Line, reason => Reason}) end. +get_role(#{role := Role}) -> + Role; +%% Before v5.4.0, +%% the field in the position of the `extra` is `desc` which is a binary for description +get_role(_Desc) -> + ?ROLE_API_DEFAULT. + +normalize_extra(Map) when is_map(Map) -> + Map; +normalize_extra(Desc) -> + #{desc => Desc, role => ?ROLE_API_DEFAULT}. + -if(?EMQX_RELEASE_EDITION == ee). check_rbac(Req, ApiKey, Role) -> case emqx_dashboard_rbac:check_rbac(Req, ApiKey, Role) of @@ -424,28 +405,3 @@ valid_role(_) -> {error, <<"Role does not exist">>}. -endif. - -maybe_migrate_table(Fields) -> - case mnesia:table_info(?APP, attributes) =:= Fields of - true -> - ok; - false -> - TransFun = fun do_table_migrate/1, - {atomic, ok} = mnesia:transform_table(?APP, TransFun, Fields, ?APP), - ok - end. - -do_table_migrate({?APP, Name, Key, Hash, Enable, Desc, ExpiredAt, CreatedAt}) -> - #?APP{ - name = Name, - api_key = Key, - api_secret_hash = Hash, - enable = Enable, - desc = Desc, - expired_at = ExpiredAt, - created_at = CreatedAt, - role = ?ROLE_API_DEFAULT, - extra = #{} - }; -do_table_migrate(#?APP{} = App) -> - App. From ec4147963311064c6dd365446f89da3758cbb5d3 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 24 Oct 2023 23:03:18 +0800 Subject: [PATCH 022/111] feat(rbac): supports setting role in API bootstrap file --- apps/emqx_management/src/emqx_mgmt_auth.erl | 26 ++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index f0d92ece7..bdb5d97fa 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -297,7 +297,7 @@ init_bootstrap_file(<<>>) -> init_bootstrap_file(File) -> case file:open(File, [read, binary]) of {ok, Dev} -> - {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), + {ok, MP} = re:compile(<<"(\.+):(\.+)(?::(\.+))?$">>, [ungreedy]), init_bootstrap_file(File, Dev, MP); {error, Reason0} -> Reason = emqx_utils:explain_posix(Reason0), @@ -327,13 +327,13 @@ init_bootstrap_file(File, Dev, MP) -> add_bootstrap_file(File, Dev, MP, Line) -> case file:read_line(Dev) of {ok, Bin} -> - case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of - {match, [[AppKey, ApiSecret]]} -> + case parse_bootstrap_line(Bin, MP) of + {ok, [AppKey, ApiSecret, Role]} -> App = #?APP{ enable = true, expired_at = infinity, - extra = #{desc => ?BOOTSTRAP_TAG, role => ?ROLE_API_DEFAULT}, + extra = #{desc => ?BOOTSTRAP_TAG, role => Role}, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), api_key = AppKey @@ -344,8 +344,7 @@ add_bootstrap_file(File, Dev, MP, Line) -> {error, Reason} -> throw(#{file => File, line => Line, content => Bin, reason => Reason}) end; - _ -> - Reason = "invalid_format", + {error, Reason} -> ?SLOG( error, #{ @@ -364,6 +363,21 @@ add_bootstrap_file(File, Dev, MP, Line) -> throw(#{file => File, line => Line, reason => Reason}) end. +parse_bootstrap_line(Bin, MP) -> + case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of + {match, [[_AppKey, _ApiSecret] = Args]} -> + {ok, Args ++ [?ROLE_API_DEFAULT]}; + {match, [[_AppKey, _ApiSecret, Role] = Args]} -> + case valid_role(Role) of + ok -> + {ok, Args}; + _Error -> + {error, {"invalid_role", Role}} + end; + _ -> + {error, "invalid_format"} + end. + get_role(#{role := Role}) -> Role; %% Before v5.4.0, From cbfd02d1b0b1bb96bc7cd7bf0646e43533a688a2 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 25 Oct 2023 18:07:00 +0800 Subject: [PATCH 023/111] refactor(ldap): merge the `ldap-bind` backend as a type for the `ldap` backend --- .../emqx_auth_ldap/include/emqx_auth_ldap.hrl | 4 - .../emqx_auth_ldap/src/emqx_auth_ldap_app.erl | 2 - apps/emqx_auth_ldap/src/emqx_authn_ldap.erl | 185 ++-------------- .../src/emqx_authn_ldap_bind.erl | 21 +- .../src/emqx_authn_ldap_bind_schema.erl | 63 ------ .../src/emqx_authn_ldap_hash.erl | 197 ++++++++++++++++++ .../src/emqx_authn_ldap_schema.erl | 61 +++++- .../test/emqx_authn_ldap_SUITE.erl | 36 ++++ .../test/emqx_authn_ldap_bind_SUITE.erl | 19 +- apps/emqx_conf/include/emqx_conf.hrl | 3 +- .../src/emqx_dashboard_sso_ldap.erl | 2 +- apps/emqx_ldap/src/emqx_ldap_bind_worker.erl | 11 +- rel/i18n/emqx_authn_ldap_bind_schema.hocon | 11 - rel/i18n/emqx_authn_ldap_schema.hocon | 15 ++ 14 files changed, 334 insertions(+), 296 deletions(-) delete mode 100644 apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl create mode 100644 apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl delete mode 100644 rel/i18n/emqx_authn_ldap_bind_schema.hocon diff --git a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl index 9cf6ac3c0..dcf0c07af 100644 --- a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl +++ b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl @@ -26,10 +26,6 @@ -define(AUTHN_BACKEND, ldap). -define(AUTHN_BACKEND_BIN, <<"ldap">>). --define(AUTHN_BACKEND_BIND, ldap_bind). --define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>). - -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}). --define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}). -endif. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 7d05faab9..5e7bd2bc6 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -25,12 +25,10 @@ start(_StartType, _StartArgs) -> ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap), ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap), - ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind), {ok, Sup} = emqx_auth_ldap_sup:start_link(), {ok, Sup}. stop(_State) -> ok = emqx_authn:deregister_provider(?AUTHN_TYPE), - ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND), ok = emqx_authz:unregister_source(?AUTHZ_TYPE), ok. diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index 975a7f828..acdd08f50 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -16,19 +16,10 @@ -module(emqx_authn_ldap). --include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("eldap/include/eldap.hrl"). -behaviour(emqx_authn_provider). -%% a compatible attribute for version 4.x --define(ISENABLED_ATTR, "isEnabled"). --define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]). -%% TODO -%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512 -%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112 - -export([ create/2, update/2, @@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) -> ignore; authenticate(#{password := undefined}, _) -> {error, bad_username_or_password}; -authenticate( - #{password := Password} = Credential, - #{ - password_attribute := PasswordAttr, - is_superuser_attribute := IsSuperuserAttr, - query_timeout := Timeout, - resource_id := ResourceId - } = State -) -> - case - emqx_resource:simple_sync_query( - ResourceId, - {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout} - ) - of - {ok, []} -> - ignore; - {ok, [Entry]} -> - is_enabled(Password, Entry, State); - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{ - resource => ResourceId, - timeout => Timeout, - reason => Reason - }), - ignore +authenticate(Credential, #{method := #{type := Type}} = State) -> + case Type of + hash -> + emqx_authn_ldap_hash:authenticate(Credential, State); + bind -> + emqx_authn_ldap_bind:authenticate(Credential, State) end. +%% it used the deprecated config form +parse_config( + #{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0 +) -> + Config = maps:without([password_attribute, is_superuser_attribute], Config0), + parse_config(Config#{ + method => #{ + type => hash, + password_attribute => PasswordAttr, + is_superuser_attribute => IsSuperuserAttr + } + }); parse_config(Config) -> - maps:with([query_timeout, password_attribute, is_superuser_attribute], Config). - -%% To compatible v4.x -is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) -> - IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"), - case emqx_authn_utils:to_bool(IsEnabled) of - true -> - ensure_password(Password, Entry, State); - _ -> - {error, user_disabled} - end. - -ensure_password( - Password, - #eldap_entry{attributes = Attributes} = Entry, - #{password_attribute := PasswordAttr} = State -) -> - case get_value(PasswordAttr, Attributes) of - undefined -> - {error, no_password}; - [LDAPPassword | _] -> - extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State) - end. - -%% RFC 2307 format password -%% https://datatracker.ietf.org/doc/html/rfc2307 -extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) -> - case - re:run( - LDAPPassword, - "{([^{}]+)}(.+)", - [{capture, all_but_first, list}, global] - ) - of - {match, [[HashTypeStr, PasswordHashStr]]} -> - case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of - {ok, HashType} -> - PasswordHash = to_binary(PasswordHashStr), - is_valid_algorithm(HashType, PasswordHash, Password, Entry, State); - _Error -> - {error, invalid_hash_type} - end; - _ -> - OnFail(LDAPPassword, Password, Entry, State) - end. - -is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) -> - case lists:member(HashType, ?VALID_ALGORITHMS) of - true -> - verify_password(HashType, PasswordHash, Password, Entry, State); - _ -> - {error, {invalid_hash_type, HashType}} - end. - -%% this password is in LDIF format which is base64 encoding -try_decode_password(LDAPPassword, Password, Entry, State) -> - case safe_base64_decode(LDAPPassword) of - {ok, Decode} -> - extract_hash_algorithm( - Decode, - Password, - fun(_, _, _, _) -> - {error, invalid_password} - end, - Entry, - State - ); - {error, Reason} -> - {error, {invalid_password, Reason}} - end. - -%% sha with salt -%% https://www.openldap.org/faq/data/cache/347.html -verify_password(ssha, PasswordData, Password, Entry, State) -> - case safe_base64_decode(PasswordData) of - {ok, <>} -> - verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State); - {ok, _} -> - {error, invalid_ssha_password}; - {error, Reason} -> - {error, {invalid_password, Reason}} - end; -verify_password( - Algorithm, - Base64HashData, - Password, - Entry, - State -) -> - verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State). - -verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) -> - PasswordHash = hash_password(Algorithm, Salt, Position, Password), - case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of - true -> - {ok, is_superuser(Entry, State)}; - _ -> - {error, bad_username_or_password} - end. - -is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) -> - Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"), - #{is_superuser => emqx_authn_utils:to_bool(Value)}. - -safe_base64_decode(Data) -> - try - {ok, base64:decode(Data)} - catch - _:Reason -> - {error, {invalid_base64_data, Reason}} - end. - -get_lower_bin_value(Key, Proplists, Default) -> - [Value | _] = get_value(Key, Proplists, [Default]), - to_binary(string:to_lower(Value)). - -to_binary(Value) -> - erlang:list_to_binary(Value). - -hash_password(Algorithm, _Salt, disable, Password) -> - hash_password(Algorithm, Password); -hash_password(Algorithm, Salt, suffix, Password) -> - hash_password(Algorithm, <>). - -hash_password(Algorithm, Data) -> - crypto:hash(Algorithm, Data). - -compare_password(hash, LDAPPasswordHash, PasswordHash) -> - emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash); -compare_password(base64, Base64HashData, PasswordHash) -> - emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)). + maps:with([query_timeout, method], Config). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl index 000d545b9..1f2af261e 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl @@ -20,32 +20,13 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("eldap/include/eldap.hrl"). --behaviour(emqx_authn_provider). - -export([ - create/2, - update/2, - authenticate/2, - destroy/1 + authenticate/2 ]). %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ - -create(_AuthenticatorID, Config) -> - emqx_authn_ldap:do_create(?MODULE, Config). - -update(Config, State) -> - emqx_authn_ldap:update(Config, State). - -destroy(State) -> - emqx_authn_ldap:destroy(State). - -authenticate(#{auth_method := _}, _) -> - ignore; -authenticate(#{password := undefined}, _) -> - {error, bad_username_or_password}; authenticate( #{password := _Password} = Credential, #{ diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl deleted file mode 100644 index 9a21766e3..000000000 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl +++ /dev/null @@ -1,63 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authn_ldap_bind_schema). - --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - --behaviour(emqx_authn_schema). - --export([ - fields/1, - desc/1, - refs/0, - select_union_member/1 -]). - -refs() -> - [?R_REF(ldap_bind)]. - -select_union_member(#{ - <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN -}) -> - refs(); -select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) -> - throw(#{ - reason => "unknown_mechanism", - expected => ?AUTHN_MECHANISM - }); -select_union_member(_) -> - undefined. - -fields(ldap_bind) -> - [ - {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)}, - {backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)}, - {query_timeout, fun query_timeout/1} - ] ++ - emqx_authn_schema:common_fields() ++ - emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts). - -desc(ldap_bind) -> - ?DESC(ldap_bind); -desc(_) -> - undefined. - -query_timeout(type) -> emqx_schema:timeout_duration_ms(); -query_timeout(desc) -> ?DESC(?FUNCTION_NAME); -query_timeout(default) -> <<"5s">>; -query_timeout(_) -> undefined. diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl new file mode 100644 index 000000000..e051e57e9 --- /dev/null +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl @@ -0,0 +1,197 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_ldap_hash). + +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("eldap/include/eldap.hrl"). + +%% a compatible attribute for version 4.x +-define(ISENABLED_ATTR, "isEnabled"). +-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]). +%% TODO +%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512 +%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112 + +-export([ + authenticate/2 +]). + +-import(proplists, [get_value/2, get_value/3]). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ +authenticate( + #{password := Password} = Credential, + #{ + method := #{ + password_attribute := PasswordAttr, + is_superuser_attribute := IsSuperuserAttr + }, + query_timeout := Timeout, + resource_id := ResourceId + } = State +) -> + case + emqx_resource:simple_sync_query( + ResourceId, + {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout} + ) + of + {ok, []} -> + ignore; + {ok, [Entry]} -> + is_enabled(Password, Entry, State); + {error, Reason} -> + ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{ + resource => ResourceId, + timeout => Timeout, + reason => Reason + }), + ignore + end. + +%% To compatible v4.x +is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) -> + IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"), + case emqx_authn_utils:to_bool(IsEnabled) of + true -> + ensure_password(Password, Entry, State); + _ -> + {error, user_disabled} + end. + +ensure_password( + Password, + #eldap_entry{attributes = Attributes} = Entry, + #{method := #{password_attribute := PasswordAttr}} = State +) -> + case get_value(PasswordAttr, Attributes) of + undefined -> + {error, no_password}; + [LDAPPassword | _] -> + extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State) + end. + +%% RFC 2307 format password +%% https://datatracker.ietf.org/doc/html/rfc2307 +extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) -> + case + re:run( + LDAPPassword, + "{([^{}]+)}(.+)", + [{capture, all_but_first, list}, global] + ) + of + {match, [[HashTypeStr, PasswordHashStr]]} -> + case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of + {ok, HashType} -> + PasswordHash = to_binary(PasswordHashStr), + is_valid_algorithm(HashType, PasswordHash, Password, Entry, State); + _Error -> + {error, invalid_hash_type} + end; + _ -> + OnFail(LDAPPassword, Password, Entry, State) + end. + +is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) -> + case lists:member(HashType, ?VALID_ALGORITHMS) of + true -> + verify_password(HashType, PasswordHash, Password, Entry, State); + _ -> + {error, {invalid_hash_type, HashType}} + end. + +%% this password is in LDIF format which is base64 encoding +try_decode_password(LDAPPassword, Password, Entry, State) -> + case safe_base64_decode(LDAPPassword) of + {ok, Decode} -> + extract_hash_algorithm( + Decode, + Password, + fun(_, _, _, _) -> + {error, invalid_password} + end, + Entry, + State + ); + {error, Reason} -> + {error, {invalid_password, Reason}} + end. + +%% sha with salt +%% https://www.openldap.org/faq/data/cache/347.html +verify_password(ssha, PasswordData, Password, Entry, State) -> + case safe_base64_decode(PasswordData) of + {ok, <>} -> + verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State); + {ok, _} -> + {error, invalid_ssha_password}; + {error, Reason} -> + {error, {invalid_password, Reason}} + end; +verify_password( + Algorithm, + Base64HashData, + Password, + Entry, + State +) -> + verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State). + +verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) -> + PasswordHash = hash_password(Algorithm, Salt, Position, Password), + case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of + true -> + {ok, is_superuser(Entry, State)}; + _ -> + {error, bad_username_or_password} + end. + +is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) -> + Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"), + #{is_superuser => emqx_authn_utils:to_bool(Value)}. + +safe_base64_decode(Data) -> + try + {ok, base64:decode(Data)} + catch + _:Reason -> + {error, {invalid_base64_data, Reason}} + end. + +get_lower_bin_value(Key, Proplists, Default) -> + [Value | _] = get_value(Key, Proplists, [Default]), + to_binary(string:to_lower(Value)). + +to_binary(Value) -> + erlang:list_to_binary(Value). + +hash_password(Algorithm, _Salt, disable, Password) -> + hash_password(Algorithm, Password); +hash_password(Algorithm, Salt, suffix, Password) -> + hash_password(Algorithm, <>). + +hash_password(Algorithm, Data) -> + crypto:hash(Algorithm, Data). + +compare_password(hash, LDAPPasswordHash, PasswordHash) -> + emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash); +compare_password(base64, Base64HashData, PasswordHash) -> + emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl index c26ca94e8..badacceea 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl @@ -29,7 +29,7 @@ ]). refs() -> - [?R_REF(ldap)]. + [?R_REF(ldap), ?R_REF(ldap_deprecated)]. select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) -> refs(); @@ -41,12 +41,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) -> select_union_member(_) -> undefined. +fields(ldap_deprecated) -> + common_fields() ++ + [ + {password_attribute, password_attribute()}, + {is_superuser_attribute, is_superuser_attribute()} + ]; fields(ldap) -> + common_fields() ++ + [ + {method, + ?HOCON( + ?UNION([?R_REF(hash_method), ?R_REF(bind_method)]), + #{desc => ?DESC(method)} + )} + ]; +fields(hash_method) -> + [ + {type, method_type(hash)}, + {password_attribute, password_attribute()}, + {is_superuser_attribute, is_superuser_attribute()} + ]; +fields(bind_method) -> + [{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts). + +common_fields() -> [ {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)}, {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, - {password_attribute, fun password_attribute/1}, - {is_superuser_attribute, fun is_superuser_attribute/1}, {query_timeout, fun query_timeout/1} ] ++ emqx_authn_schema:common_fields() ++ @@ -54,18 +76,35 @@ fields(ldap) -> desc(ldap) -> ?DESC(ldap); +desc(ldap_deprecated) -> + ?DESC(ldap_deprecated); +desc(hash_method) -> + ?DESC(hash_method); +desc(bind_method) -> + ?DESC(bind_method); desc(_) -> undefined. -password_attribute(type) -> string(); -password_attribute(desc) -> ?DESC(?FUNCTION_NAME); -password_attribute(default) -> <<"userPassword">>; -password_attribute(_) -> undefined. +method_type(Type) -> + ?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}). -is_superuser_attribute(type) -> string(); -is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME); -is_superuser_attribute(default) -> <<"isSuperuser">>; -is_superuser_attribute(_) -> undefined. +password_attribute() -> + ?HOCON( + string(), + #{ + desc => ?DESC(?FUNCTION_NAME), + default => <<"userPassword">> + } + ). + +is_superuser_attribute() -> + ?HOCON( + string(), + #{ + desc => ?DESC(?FUNCTION_NAME), + default => <<"isSuperuser">> + } + ). query_timeout(type) -> emqx_schema:timeout_duration_ms(); query_timeout(desc) -> ?DESC(?FUNCTION_NAME); diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index 63bceee85..2aa1c5c96 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -70,6 +70,29 @@ end_per_suite(Config) -> %% Tests %%------------------------------------------------------------------------------ +t_create_with_deprecated_cfg(_Config) -> + AuthConfig = deprecated_raw_ldap_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators( + ?GLOBAL + ), + ?assertMatch( + #{ + method := #{ + type := hash, + is_superuser_attribute := _, + password_attribute := "not_the_default_value" + } + }, + State + ), + emqx_authn_test_lib:delete_config(?ResourceID). + t_create(_Config) -> AuthConfig = raw_ldap_auth_config(), @@ -225,6 +248,19 @@ raw_ldap_auth_config() -> <<"pool_size">> => 8 }. +deprecated_raw_ldap_auth_config() -> + #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"ldap">>, + <<"server">> => ldap_server(), + <<"is_superuser_attribute">> => <<"isSuperuser">>, + <<"password_attribute">> => <<"not_the_default_value">>, + <<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>, + <<"username">> => <<"cn=root,dc=emqx,dc=io">>, + <<"password">> => <<"public">>, + <<"pool_size">> => 8 + }. + user_seeds() -> New = fun(Username, Password, Result) -> #{ diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index 1f390264b..d2b3c371c 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -27,7 +27,7 @@ -define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>). -define(PATH, [authentication]). --define(ResourceID, <<"password_based:ldap_bind">>). +-define(ResourceID, <<"password_based:ldap">>). all() -> emqx_common_test_helpers:all(?MODULE). @@ -78,7 +78,7 @@ t_create(_Config) -> {create_authenticator, ?GLOBAL, AuthConfig} ), - {ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + {ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL), emqx_authn_test_lib:delete_config(?ResourceID). t_create_invalid(_Config) -> @@ -146,10 +146,10 @@ t_destroy(_Config) -> {create_authenticator, ?GLOBAL, AuthConfig} ), - {ok, [#{provider := emqx_authn_ldap_bind, state := State}]} = + {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), - {ok, _} = emqx_authn_ldap_bind:authenticate( + {ok, _} = emqx_authn_ldap:authenticate( #{ username => <<"mqttuser0001">>, password => <<"mqttuser0001">> @@ -165,7 +165,7 @@ t_destroy(_Config) -> % Authenticator should not be usable anymore ?assertMatch( ignore, - emqx_authn_ldap_bind:authenticate( + emqx_authn_ldap:authenticate( #{ username => <<"mqttuser0001">>, password => <<"mqttuser0001">> @@ -199,7 +199,7 @@ t_update(_Config) -> % We update with config with correct query, provider should update and work properly {ok, _} = emqx:update_config( ?PATH, - {update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig} + {update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig} ), {ok, _} = emqx_access_control:authenticate( @@ -218,14 +218,17 @@ t_update(_Config) -> raw_ldap_auth_config() -> #{ <<"mechanism">> => <<"password_based">>, - <<"backend">> => <<"ldap_bind">>, + <<"backend">> => <<"ldap">>, <<"server">> => ldap_server(), <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>, <<"filter">> => <<"(uid=${username})">>, <<"username">> => <<"cn=root,dc=emqx,dc=io">>, <<"password">> => <<"public">>, <<"pool_size">> => 8, - <<"bind_password">> => <<"${password}">> + <<"method">> => #{ + <<"type">> => <<"bind">>, + <<"bind_password">> => <<"${password}">> + } }. user_seeds() -> diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index a758681ff..6c4a89fb8 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -58,8 +58,7 @@ emqx_authn_http_schema, emqx_authn_jwt_schema, emqx_authn_scram_mnesia_schema, - emqx_authn_ldap_schema, - emqx_authn_ldap_bind_schema + emqx_authn_ldap_schema ]). -define(EE_AUTHN_PROVIDER_SCHEMA_MODS, [ diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl index 499e24c5b..b6bdcf744 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl @@ -92,7 +92,7 @@ parse_config(Config0) -> %% In this feature, the `bind_password` is fixed, so it should conceal from the swagger, %% but the connector still needs it, hence we should add it back here ensure_bind_password(Config) -> - Config#{bind_password => <<"${password}">>}. + Config#{method => #{type => bind, bind_password => <<"${password}">>}}. adjust_ldap_fields(Fields) -> lists:map(fun adjust_ldap_field/1, Fields). diff --git a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl index 722e79006..834cbac5a 100644 --- a/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl +++ b/apps/emqx_ldap/src/emqx_ldap_bind_worker.erl @@ -35,7 +35,7 @@ %% =================================================================== -spec on_start(binary(), hoconsc:config(), proplists:proplist(), map()) -> {ok, binary(), map()} | {error, _}. -on_start(InstId, #{bind_password := _} = Config, Options, State) -> +on_start(InstId, #{method := #{bind_password := _}} = Config, Options, State) -> PoolName = pool_name(InstId), ?SLOG(info, #{ msg => "starting_ldap_bind_worker", @@ -108,15 +108,10 @@ on_query( connect(Conf) -> emqx_ldap:connect(Conf). -prepare_template(Config, State) -> - do_prepare_template(maps:to_list(maps:with([bind_password], Config)), State). - -do_prepare_template([{bind_password, V} | T], State) -> +prepare_template(#{method := #{bind_password := V}}, State) -> %% This is sensitive data %% to reduce match cases, here we reuse the existing sensitive filter key: bind_password - do_prepare_template(T, State#{bind_password => emqx_placeholder:preproc_tmpl(V)}); -do_prepare_template([], State) -> - State. + State#{bind_password => emqx_placeholder:preproc_tmpl(V)}. pool_name(InstId) -> <>. diff --git a/rel/i18n/emqx_authn_ldap_bind_schema.hocon b/rel/i18n/emqx_authn_ldap_bind_schema.hocon deleted file mode 100644 index b0f20aa10..000000000 --- a/rel/i18n/emqx_authn_ldap_bind_schema.hocon +++ /dev/null @@ -1,11 +0,0 @@ -emqx_authn_ldap_bind_schema { - -ldap_bind.desc: -"""Configuration of authenticator using the LDAP bind operation as the authentication method.""" - -query_timeout.desc: -"""Timeout for the LDAP query.""" - -query_timeout.label: -"""Query Timeout""" -} diff --git a/rel/i18n/emqx_authn_ldap_schema.hocon b/rel/i18n/emqx_authn_ldap_schema.hocon index 41f57ffbc..40e2882ba 100644 --- a/rel/i18n/emqx_authn_ldap_schema.hocon +++ b/rel/i18n/emqx_authn_ldap_schema.hocon @@ -3,6 +3,9 @@ emqx_authn_ldap_schema { ldap.desc: """Configuration of authenticator using LDAP as authentication data source.""" +ldap_deprecated.desc: +"""This is a deprecated form, you should avoid using it.""" + password_attribute.desc: """Indicates which attribute is used to represent the user's password.""" @@ -21,4 +24,16 @@ query_timeout.desc: query_timeout.label: """Query Timeout""" +hash_method.desc: +"""Authenticate by comparing the hashed password which was provided by the `password attribute`.""" + +bind_method.desc: +"""Authenticate by the LDAP bind operation.""" + +method.desc: +"""Authentication method.""" + +method_type.desc: +"""Authentication method type.""" + } From 44b4205561d63ac0440d776d040dba2d2c647de8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 26 Oct 2023 14:37:14 +0700 Subject: [PATCH 024/111] fix(secret): do not treat missing file secrets as config error They are intended to be used mostly in the context of resources, which have their own feedback mechanism: statuses, retries, etc. Also turn the error into a throw exception, so that it can be interpreted as a regular error condition, for example by the resource manager. --- apps/emqx/src/emqx_schema_secret.erl | 15 ++++++-------- .../test/emqx_bridge_mqtt_SUITE.erl | 20 ++++++++++--------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/apps/emqx/src/emqx_schema_secret.erl b/apps/emqx/src/emqx_schema_secret.erl index aa2cbcc84..865d0eac9 100644 --- a/apps/emqx/src/emqx_schema_secret.erl +++ b/apps/emqx/src/emqx_schema_secret.erl @@ -75,14 +75,7 @@ convert_secret(Secret, #{}) -> -spec wrap(source()) -> emqx_secret:t(t()). wrap(Source) -> - try - _Secret = load(Source), - emqx_secret:wrap(?MODULE, load, Source) - catch - error:Reason -> - % NOTE: This should be a term serializable as JSON value. - throw(emqx_utils:format(Reason)) - end. + emqx_secret:wrap(?MODULE, load, Source). -spec source(emqx_secret:t(t())) -> source(). source(Secret) when is_function(Secret) -> @@ -103,5 +96,9 @@ load_file(Filename) -> {ok, Secret} -> string:trim(Secret, trailing, [$\n]); {error, Reason} -> - error({inaccessible_secret_file, Reason}, [Filename]) + throw(#{ + msg => failed_to_read_secret_file, + path => Filename, + reason => emqx_utils:explain_posix(Reason) + }) end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 6b5cc86da..1776ae236 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -230,16 +230,18 @@ t_conf_bridge_authn_passfile(Config) -> ?assertReceive( {authenticate, #{username := Username2, password := Password2}} ), - {ok, 400, #{<<"message">> := Message}} = request_json( - post, - uri(["bridges"]), - ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ - <<"name">> => <<"t_conf_bridge_authn_no_passfile">> - } - ), ?assertMatch( - #{<<"reason">> := <<"{inaccessible_secret_file,enoent}">>}, - emqx_utils_json:decode(Message) + {ok, 201, #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"#{msg => failed_to_read_secret_file", _/bytes>> + }}, + request_json( + post, + uri(["bridges"]), + ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ + <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + } + ) ). hook_authenticate() -> From b5411da770b1251bc05cfc61215777286748a285 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 7 Jun 2023 10:22:14 +0800 Subject: [PATCH 025/111] refactor: subscribe process to fix shared-sub --- README-CN.md | 2 +- apps/emqx/include/emqx.hrl | 3 - apps/emqx/include/emqx_mqtt.hrl | 18 +- apps/emqx/src/emqx_broker.erl | 88 +++-- apps/emqx/src/emqx_channel.erl | 305 +++++++++++------- apps/emqx/src/emqx_mountpoint.erl | 47 ++- apps/emqx/src/emqx_mqtt_caps.erl | 15 +- apps/emqx/src/emqx_reason_codes.erl | 1 + apps/emqx/src/emqx_session.erl | 14 +- apps/emqx/src/emqx_session_mem.erl | 2 +- apps/emqx/src/emqx_topic.erl | 89 +++-- apps/emqx/src/emqx_types.erl | 9 +- apps/emqx/test/emqx_broker_SUITE.erl | 28 +- apps/emqx/test/emqx_channel_SUITE.erl | 11 +- apps/emqx/test/emqx_mountpoint_SUITE.erl | 39 +++ apps/emqx/test/emqx_mqtt_caps_SUITE.erl | 4 +- apps/emqx/test/emqx_shared_sub_SUITE.erl | 189 +++++++---- apps/emqx/test/emqx_topic_SUITE.erl | 12 +- .../src/emqx_bridge_mqtt_connector.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_handler.erl | 14 +- .../test/props/prop_exhook_hooks.erl | 1 + .../src/emqx_mgmt_api_clients.erl | 11 +- .../src/emqx_mgmt_api_subscriptions.erl | 41 +-- .../src/emqx_mgmt_api_topics.erl | 18 +- apps/emqx_modules/src/emqx_rewrite.erl | 3 +- apps/emqx_retainer/src/emqx_retainer.erl | 3 +- .../emqx_rule_engine/src/emqx_rule_events.erl | 8 +- changes/ce/fix-10976.en.md | 2 + 28 files changed, 636 insertions(+), 343 deletions(-) create mode 100644 changes/ce/fix-10976.en.md diff --git a/README-CN.md b/README-CN.md index 8c6f8d8c3..f989b9bed 100644 --- a/README-CN.md +++ b/README-CN.md @@ -77,7 +77,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl 优雅的跨平台 MQTT 5.0 客户端工具，提供了桌面端、命令行、Web 三种版本，帮助您更快的开发和调试 MQTT 服务和应用。 -- [车联网平台搭建从入门到精通 ](https://www.emqx.com/zh/blog/category/internet-of-vehicles) +- [车联网平台搭建从入门到精通](https://www.emqx.com/zh/blog/category/internet-of-vehicles) 结合 EMQ 在车联网领域的实践经验，从协议选择等理论知识，到平台架构设计等实战操作，分享如何搭建一个可靠、高效、符合行业场景需求的车联网平台。 diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 664ec5803..3650488dd 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -39,9 +39,6 @@ %% System topic -define(SYSTOP, <<"$SYS/">>). -%% Queue topic --define(QUEUE, <<"$queue/">>). - %%-------------------------------------------------------------------- %% alarms %%-------------------------------------------------------------------- diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 4d0188f71..93c70a6e1 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -55,6 +55,17 @@ %% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3] -define(MAX_TOPIC_LEN, 65535). +%%-------------------------------------------------------------------- +%% MQTT Share-Sub Internal +%%-------------------------------------------------------------------- + +-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}). + +%% guards +-define(IS_TOPIC(T), + (is_binary(T) orelse is_record(T, share)) +). + %%-------------------------------------------------------------------- %% MQTT QoS Levels %%-------------------------------------------------------------------- @@ -661,13 +672,8 @@ end). -define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}). -define(SHARE, "$share"). +-define(QUEUE, "$queue"). -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). --define(IS_SHARE(Topic), - case Topic of - <> -> true; - _ -> false - end -). -define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty). -define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 403e3757f..cc9cb98a6 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -118,18 +118,20 @@ create_tabs() -> %% Subscribe API %%------------------------------------------------------------------------------ --spec subscribe(emqx_types:topic()) -> ok. -subscribe(Topic) when is_binary(Topic) -> +-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok. +subscribe(Topic) when ?IS_TOPIC(Topic) -> subscribe(Topic, undefined). --spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok. -subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) -> +-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) -> + ok. +subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); -subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> +subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). --spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok. -subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> +-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) -> + ok. +subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), _ = emqx_trace:subscribe(Topic, SubId, SubOpts), SubPid = self(), @@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) -> with_subid(SubId, SubOpts) -> maps:put(subid, SubId, SubOpts). -%% @private do_subscribe(Topic, SubPid, SubOpts) -> true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), - Group = maps:get(share, SubOpts, undefined), - do_subscribe(Group, Topic, SubPid, SubOpts). + do_subscribe2(Topic, SubPid, SubOpts). -do_subscribe(undefined, Topic, SubPid, SubOpts) -> +do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) -> + %% FIXME: subscribe shard bug + %% https://emqx.atlassian.net/browse/EMQX-10214 case emqx_broker_helper:get_sub_shard(SubPid, Topic) of 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), @@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) -> true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}), call(pick({Topic, I}), {subscribe, Topic, I}) end; -%% Shared subscription -do_subscribe(Group, Topic, SubPid, SubOpts) -> +do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when + is_binary(RealTopic) +-> true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}), - emqx_shared_sub:subscribe(Group, Topic, SubPid). + emqx_shared_sub:subscribe(Group, RealTopic, SubPid). %%-------------------------------------------------------------------- %% Unsubscribe API %%-------------------------------------------------------------------- --spec unsubscribe(emqx_types:topic()) -> ok. -unsubscribe(Topic) when is_binary(Topic) -> +-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok. +unsubscribe(Topic) when ?IS_TOPIC(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {Topic, SubPid}) of [{_, SubOpts}] -> - _ = emqx_broker_helper:reclaim_seq(Topic), _ = emqx_trace:unsubscribe(Topic, SubOpts), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok end. +-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) -> + ok. do_unsubscribe(Topic, SubPid, SubOpts) -> true = ets:delete(?SUBOPTION, {Topic, SubPid}), true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), - Group = maps:get(share, SubOpts, undefined), - do_unsubscribe(Group, Topic, SubPid, SubOpts). + do_unsubscribe2(Topic, SubPid, SubOpts). -do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> +-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) -> + ok. +do_unsubscribe2(Topic, SubPid, SubOpts) when + is_binary(Topic), is_pid(SubPid), is_map(SubOpts) +-> + _ = emqx_broker_helper:reclaim_seq(Topic), case maps:get(shard, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), @@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; -do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> +do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when + is_binary(Group), is_binary(Topic), is_pid(SubPid) +-> emqx_shared_sub:unsubscribe(Group, Topic, SubPid). %%-------------------------------------------------------------------- @@ -306,7 +316,9 @@ aggre([], true, Acc) -> lists:usort(Acc). %% @doc Forward message to another node. --spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) -> +-spec forward( + node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async +) -> emqx_types:deliver_result(). forward(Node, To, Delivery, async) -> true = emqx_broker_proto_v1:forward_async(Node, To, Delivery), @@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) -> Result end. --spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result(). +-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) -> + emqx_types:deliver_result(). dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) -> case emqx:is_running() of true -> @@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) -> end. -compile({inline, [subscribers/1]}). --spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) -> +-spec subscribers( + emqx_types:topic() + | emqx_types:share() + | {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()} +) -> [pid()]. subscribers(Topic) when is_binary(Topic) -> lookup_value(?SUBSCRIBER, Topic, []); @@ -372,7 +389,7 @@ subscriber_down(SubPid) -> SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), true = ets:delete(?SUBOPTION, {Topic, SubPid}), - do_unsubscribe(undefined, Topic, SubPid, SubOpts); + do_unsubscribe2(Topic, SubPid, SubOpts); undefined -> ok end @@ -386,7 +403,7 @@ subscriber_down(SubPid) -> %%-------------------------------------------------------------------- -spec subscriptions(pid() | emqx_types:subid()) -> - [{emqx_types:topic(), emqx_types:subopts()}]. + [{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}]. subscriptions(SubPid) when is_pid(SubPid) -> [ {Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})} @@ -400,20 +417,22 @@ subscriptions(SubId) -> [] end. --spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()]. +-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()]. subscriptions_via_topic(Topic) -> MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}], ets:select(?SUBOPTION, MatchSpec). --spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean(). +-spec subscribed( + pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share() +) -> boolean(). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {Topic, SubPid}); subscribed(SubId, Topic) when ?IS_SUBID(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {Topic, SubPid}). --spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()). -get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> +-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()). +get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) -> lookup_value(?SUBOPTION, {Topic, SubPid}); get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> case emqx_broker_helper:lookup_subpid(SubId) of @@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> undefined end. --spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean(). +-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean(). set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> set_subopts(self(), Topic, NewOpts). @@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) -> false end. --spec topics() -> [emqx_types:topic()]. +-spec topics() -> [emqx_types:topic() | emqx_types:share()]. topics() -> emqx_router:topics(). @@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- --spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result(). +-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) -> + emqx_types:deliver_result(). do_dispatch(Topic, #delivery{message = Msg}) -> DispN = lists:foldl( fun(Sub, N) -> @@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) -> {ok, DispN} end. +%% Donot dispatch to share subscriber here. +%% we do it in `emqx_shared_sub.erl` with configured strategy do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> case erlang:is_process_alive(SubPid) of true -> diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 4f6d5ac6f..61b31c6e1 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -476,60 +476,27 @@ handle_in( ok = emqx_metrics:inc('packets.pubcomp.missed'), {ok, Channel} end; -handle_in( - SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), - Channel = #channel{clientinfo = ClientInfo} -) -> - case emqx_packet:check(SubPkt) of - ok -> - TopicFilters0 = parse_topic_filters(TopicFilters), - TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0), - TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel), - HasAuthzDeny = lists:any( - fun({_TopicFilter, ReasonCode}) -> - ReasonCode =:= ?RC_NOT_AUTHORIZED - end, - TupleTopicFilters0 - ), - DenyAction = emqx:get_config([authorization, deny_action], ignore), - case DenyAction =:= disconnect andalso HasAuthzDeny of - true -> - handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel); - false -> - TopicFilters2 = [ - TopicFilter - || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0 - ], - TopicFilters3 = run_hooks( - 'client.subscribe', - [ClientInfo, Properties], - TopicFilters2 - ), - {TupleTopicFilters1, NChannel} = process_subscribe( - TopicFilters3, - Properties, - Channel - ), - TupleTopicFilters2 = - lists:foldl( - fun - ({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) -> - Key = {Topic, maps:without([deny_subscription], Opts)}, - lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR}); - (Tuple = {Key, _Value}, Acc) -> - lists:keyreplace(Key, 1, Acc, Tuple) - end, - TupleTopicFilters0, - TupleTopicFilters1 - ), - ReasonCodes2 = [ - ReasonCode - || {_TopicFilter, ReasonCode} <- TupleTopicFilters2 - ], - handle_out(suback, {PacketId, ReasonCodes2}, NChannel) - end; - {error, ReasonCode} -> - handle_out(disconnect, ReasonCode, Channel) +handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) -> + Pipe = pipeline( + [ + fun check_subscribe/2, + fun enrich_subscribe/2, + %% TODO && FIXME (EMQX-10786): mount topic before authz check. + fun check_sub_authzs/2, + fun check_sub_caps/2 + ], + SubPkt, + Channel0 + ), + case Pipe of + {ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} -> + {TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel), + ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC), + handle_out(suback, {PacketId, ReasonCodes}, NChannel); + {error, {disconnect, RC}, Channel} -> + %% funcs in pipeline always cause action: `disconnect` + %% And Only one ReasonCode in DISCONNECT packet + handle_out(disconnect, RC, Channel) end; handle_in( Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), @@ -540,7 +507,7 @@ handle_in( TopicFilters1 = run_hooks( 'client.unsubscribe', [ClientInfo, Properties], - parse_topic_filters(TopicFilters) + parse_raw_topic_filters(TopicFilters) ), {ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel), handle_out(unsuback, {PacketId, ReasonCodes}, NChannel); @@ -782,32 +749,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) -> %% Process Subscribe %%-------------------------------------------------------------------- --compile({inline, [process_subscribe/3]}). -process_subscribe(TopicFilters, SubProps, Channel) -> - process_subscribe(TopicFilters, SubProps, Channel, []). +process_subscribe(TopicFilters, Channel) -> + process_subscribe(TopicFilters, Channel, []). -process_subscribe([], _SubProps, Channel, Acc) -> +process_subscribe([], Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) -> - case check_sub_caps(TopicFilter, SubOpts, Channel) of - ok -> - {ReasonCode, NChannel} = do_subscribe( - TopicFilter, - SubOpts#{sub_props => SubProps}, - Channel - ), - process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]); - {error, ReasonCode} -> - ?SLOG( - warning, - #{ - msg => "cannot_subscribe_topic_filter", - reason => emqx_reason_codes:name(ReasonCode) - }, - #{topic => TopicFilter} - ), - process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) - end. +process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) -> + {NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel), + process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]). do_subscribe( TopicFilter, @@ -818,11 +767,13 @@ do_subscribe( session = Session } ) -> + %% TODO && FIXME (EMQX-10786): mount topic before authz check. NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter), - NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel), - case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of + case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of {ok, NSession} -> - {QoS, Channel#channel{session = NSession}}; + %% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now + RC = QoS, + {RC, Channel#channel{session = NSession}}; {error, RC} -> ?SLOG( warning, @@ -835,6 +786,30 @@ do_subscribe( {RC, Channel} end. +gen_reason_codes(TFChecked, TFSubedWitNhRC) -> + do_gen_reason_codes([], TFChecked, TFSubedWitNhRC). + +%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2 +%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and +%% NRC should override the initial RC. +do_gen_reason_codes(Acc, [], []) -> + lists:reverse(Acc); +do_gen_reason_codes( + Acc, + [{_TF, ?RC_SUCCESS} | RestCheckedTF], + [{_TF, NRC} | RestTFWithNRC] +) -> + %% will passing through `process_subscribe/2` + %% use NRC to override IintialRC + do_gen_reason_codes([NRC | Acc], RestCheckedTF, RestTFWithNRC); +do_gen_reason_codes( + Acc, + [{_TF, InitialRC} | RestChecked], + RestTFWithNRC +) -> + %% InitialRC is not `RC_SUCCESS`, use it. + do_gen_reason_codes([InitialRC | Acc], RestChecked, RestTFWithNRC). + %%-------------------------------------------------------------------- %% Process Unsubscribe %%-------------------------------------------------------------------- @@ -1213,13 +1188,8 @@ handle_call(Req, Channel) -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. handle_info({subscribe, TopicFilters}, Channel) -> - {_, NChannel} = lists:foldl( - fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> - do_subscribe(TopicFilter, SubOpts, ChannelAcc) - end, - {[], Channel}, - parse_topic_filters(TopicFilters) - ), + NTopicFilters = enrich_subscribe(TopicFilters, Channel), + {_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel), {ok, NChannel}; handle_info({unsubscribe, TopicFilters}, Channel) -> {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), @@ -1857,49 +1827,156 @@ check_pub_caps( ) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}). +%%-------------------------------------------------------------------- +%% Check Subscribe Packet + +check_subscribe(SubPkt, _Channel) -> + case emqx_packet:check(SubPkt) of + ok -> ok; + {error, RC} -> {error, {disconnect, RC}} + end. + %%-------------------------------------------------------------------- %% Check Sub Authorization -check_sub_authzs(TopicFilters, Channel) -> - check_sub_authzs(TopicFilters, Channel, []). - check_sub_authzs( - [TopicFilter = {Topic, _} | More], - Channel = #channel{clientinfo = ClientInfo}, - Acc + ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0), + Channel = #channel{clientinfo = ClientInfo} ) -> + CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo), + HasAuthzDeny = lists:any( + fun({{_TopicFilter, _SubOpts}, ReasonCode}) -> + ReasonCode =:= ?RC_NOT_AUTHORIZED + end, + CheckResult + ), + DenyAction = emqx:get_config([authorization, deny_action], ignore), + case DenyAction =:= disconnect andalso HasAuthzDeny of + true -> + {error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel}; + false -> + {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel} + end. + +do_check_sub_authzs(TopicFilters, ClientInfo) -> + do_check_sub_authzs(ClientInfo, TopicFilters, []). + +do_check_sub_authzs(_ClientInfo, [], Acc) -> + lists:reverse(Acc); +do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) -> + %% subsclibe authz check only cares the real topic filter when shared-sub + %% e.g. only check <<"t/#">> for <<"$share/g/t/#">> Action = authz_action(TopicFilter), - case emqx_access_control:authorize(ClientInfo, Action, Topic) of + case + emqx_access_control:authorize( + ClientInfo, + Action, + emqx_topic:get_shared_real_topic(Topic) + ) + of + %% TODO: support maximum QoS granted + %% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7] + %% Not implemented yet: + %% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]); allow -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); + do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]); deny -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) - end; -check_sub_authzs([], _Channel, Acc) -> - lists:reverse(Acc). + do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + end. %%-------------------------------------------------------------------- %% Check Sub Caps -check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) -> - emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts). +check_sub_caps( + ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters), + Channel = #channel{clientinfo = ClientInfo} +) -> + CheckResult = do_check_sub_caps(ClientInfo, TopicFilters), + {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}. + +do_check_sub_caps(ClientInfo, TopicFilters) -> + do_check_sub_caps(ClientInfo, TopicFilters, []). + +do_check_sub_caps(_ClientInfo, [], Acc) -> + lists:reverse(Acc); +do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) -> + case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of + ok -> + do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]); + {error, NRC} -> + ?SLOG( + warning, + #{ + msg => "cannot_subscribe_topic_filter", + reason => emqx_reason_codes:name(NRC) + }, + #{topic => Topic} + ), + do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc]) + end; +do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) -> + do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]). %%-------------------------------------------------------------------- -%% Enrich SubId +%% Run Subscribe Hooks -enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) -> - [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters]; -enrich_subopts_subid(_Properties, TopicFilters) -> - TopicFilters. +run_sub_hooks( + ?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0), + _Channel = #channel{clientinfo = ClientInfo} +) -> + TopicFilters = [ + TopicFilter + || {TopicFilter, ?RC_SUCCESS} <- TopicFilters0 + ], + _NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters). %%-------------------------------------------------------------------- %% Enrich SubOpts -enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) -> - SubOpts; -enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) -> +%% for api subscribe without sub-authz check and sub-caps check. +enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) -> + do_enrich_subscribe(#{}, TopicFilters, Channel); +%% for mqtt clients sent subscribe packet. +enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) -> + NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel), + {ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}. + +do_enrich_subscribe(Properties, TopicFilters, Channel) -> + _NTopicFilters = run_fold( + [ + %% TODO: do try catch with reason code here + fun(TFs, _) -> parse_raw_topic_filters(TFs) end, + fun enrich_subopts_subid/2, + fun enrich_subopts_porps/2, + fun enrich_subopts_flags/2 + ], + TopicFilters, + #{sub_props => Properties, channel => Channel} + ). + +enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) -> + [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters]; +enrich_subopts_subid(TopicFilters, _State) -> + TopicFilters. + +enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) -> + [{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters]. + +enrich_subopts_flags(TopicFilters, #{channel := Channel}) -> + do_enrich_subopts_flags(TopicFilters, Channel). + +do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) -> + [{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters]; +do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) -> + Rap = flag(IsBridge), NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)), - SubOpts#{rap => flag(IsBridge), nl => NL}. + [ + {Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}} + || {Topic, SubOpts} <- TopicFilters + ]. + +merge_default_subopts(SubOpts) -> + maps:merge(?DEFAULT_SUBOPTS, SubOpts). %%-------------------------------------------------------------------- %% Enrich ConnAck Caps @@ -2089,8 +2166,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) -> %%-------------------------------------------------------------------- %% Parse Topic Filters --compile({inline, [parse_topic_filters/1]}). -parse_topic_filters(TopicFilters) -> +%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _] +parse_raw_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_mountpoint.erl b/apps/emqx/src/emqx_mountpoint.erl index 5b5dac954..c19736690 100644 --- a/apps/emqx/src/emqx_mountpoint.erl +++ b/apps/emqx/src/emqx_mountpoint.erl @@ -17,6 +17,7 @@ -module(emqx_mountpoint). -include("emqx.hrl"). +-include("emqx_mqtt.hrl"). -include("emqx_placeholder.hrl"). -include("types.hrl"). @@ -34,38 +35,54 @@ -spec mount(maybe(mountpoint()), Any) -> Any when Any :: emqx_types:topic() + | emqx_types:share() | emqx_types:message() | emqx_types:topic_filters(). mount(undefined, Any) -> Any; -mount(MountPoint, Topic) when is_binary(Topic) -> - prefix(MountPoint, Topic); -mount(MountPoint, Msg = #message{topic = Topic}) -> - Msg#message{topic = prefix(MountPoint, Topic)}; +mount(MountPoint, Topic) when ?IS_TOPIC(Topic) -> + prefix_maybe_share(MountPoint, Topic); +mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) -> + Msg#message{topic = prefix_maybe_share(MountPoint, Topic)}; mount(MountPoint, TopicFilters) when is_list(TopicFilters) -> - [{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters]. + [{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters]. -%% @private --compile({inline, [prefix/2]}). -prefix(MountPoint, Topic) -> - <>. +-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when + Any :: + emqx_types:topic() + | emqx_types:share(). +prefix_maybe_share(MountPoint, Topic) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + <>; +prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + #share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}. -spec unmount(maybe(mountpoint()), Any) -> Any when Any :: emqx_types:topic() + | emqx_types:share() | emqx_types:message(). unmount(undefined, Any) -> Any; -unmount(MountPoint, Topic) when is_binary(Topic) -> +unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) -> + unmount_maybe_share(MountPoint, Topic); +unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) -> + Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}. + +unmount_maybe_share(MountPoint, Topic) when + is_binary(MountPoint) andalso is_binary(Topic) +-> case string:prefix(Topic, MountPoint) of nomatch -> Topic; Topic1 -> Topic1 end; -unmount(MountPoint, Msg = #message{topic = Topic}) -> - case string:prefix(Topic, MountPoint) of - nomatch -> Msg; - Topic1 -> Msg#message{topic = Topic1} - end. +unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}. -spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()). replvar(undefined, _Vars) -> diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index 11f495dbd..5cf10691d 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) -> -spec check_sub( emqx_types:clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts() ) -> ok_or_error(emqx_types:reason_code()). check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) -> Caps = emqx_config:get_zone_conf(Zone, [mqtt]), Flags = #{ + %% TODO: qos check + %% (max_qos_allowed, Map) -> + %% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2), topic_levels => emqx_topic:levels(Topic), is_wildcard => emqx_topic:wildcard(Topic), - is_shared => maps:is_key(share, SubOpts), + is_shared => erlang:is_record(Topic, share), is_exclusive => maps:get(is_exclusive, SubOpts, false) }, do_check_sub(Flags, Caps, ClientInfo, Topic). @@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}; do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) -> {error, ?RC_TOPIC_FILTER_INVALID}; -do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) -> +do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when + is_binary(Topic) +-> case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of deny -> {error, ?RC_QUOTA_EXCEEDED}; _ -> ok end; +%% for max_qos_allowed +%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2 +%% do_check_sub(_, _) -> +%% {ok, RC}; do_check_sub(_Flags, _Caps, _, _) -> ok. diff --git a/apps/emqx/src/emqx_reason_codes.erl b/apps/emqx/src/emqx_reason_codes.erl index 77a8c1be2..543a62216 100644 --- a/apps/emqx/src/emqx_reason_codes.erl +++ b/apps/emqx/src/emqx_reason_codes.erl @@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER; compat(connack, 16#9F) -> ?CONNACK_SERVER; compat(suback, Code) when Code =< ?QOS_2 -> Code; compat(suback, Code) when Code >= 16#80 -> 16#80; +%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1 compat(unsuback, _Code) -> undefined; compat(_Other, _Code) -> undefined. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 8bdd47392..e79c30f4a 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -259,7 +259,7 @@ destroy(Session) -> -spec subscribe( clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts(), t() ) -> @@ -279,7 +279,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) -> -spec unsubscribe( clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts(), t() ) -> @@ -409,6 +409,16 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> [Msg | enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session)] end. +enrich_deliver( + ClientInfo, + {deliver, Topic, Msg = #message{headers = #{redispatch_to := {Group, Topic}}}}, + UpgradeQoS, + Session +) -> + %% Only QoS_1 and QoS_2 messages added `redispatch_to` header + %% For QoS 0 message, send it as regular dispatch + Deliver = {deliver, emqx_topic:make_shared_record(Group, Topic), Msg}, + enrich_deliver(ClientInfo, Deliver, UpgradeQoS, Session); enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> SubOpts = ?IMPL(Session):get_subscription(Topic, Session), enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS). diff --git a/apps/emqx/src/emqx_session_mem.erl b/apps/emqx/src/emqx_session_mem.erl index e72feffd5..8279953c1 100644 --- a/apps/emqx/src/emqx_session_mem.erl +++ b/apps/emqx/src/emqx_session_mem.erl @@ -314,7 +314,7 @@ unsubscribe( {error, ?RC_NO_SUBSCRIPTION_EXISTED} end. --spec get_subscription(emqx_types:topic(), session()) -> +-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) -> emqx_types:subopts() | undefined. get_subscription(Topic, #session{subscriptions = Subs}) -> maps:get(Topic, Subs, undefined). diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 6d232c68d..20dfd4316 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -36,9 +36,16 @@ parse/2 ]). +-export([ + maybe_format_share/1, + get_shared_real_topic/1, + make_shared_record/2 +]). + -type topic() :: emqx_types:topic(). -type word() :: emqx_types:word(). -type words() :: emqx_types:words(). +-type share() :: emqx_types:share(). %% Guards -define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST), @@ -50,7 +57,9 @@ %%-------------------------------------------------------------------- %% @doc Is wildcard topic? --spec wildcard(topic() | words()) -> true | false. +-spec wildcard(topic() | share() | words()) -> true | false. +wildcard(#share{topic = Topic}) when is_binary(Topic) -> + wildcard(Topic); wildcard(Topic) when is_binary(Topic) -> wildcard(words(Topic)); wildcard([]) -> @@ -64,7 +73,7 @@ wildcard([_H | T]) -> %% @doc Match Topic name with filter. -spec match(Name, Filter) -> boolean() when - Name :: topic() | words(), + Name :: topic() | share() | words(), Filter :: topic() | words(). match(<<$$, _/binary>>, <<$+, _/binary>>) -> false; @@ -72,6 +81,8 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) -> false; match(Name, Filter) when is_binary(Name), is_binary(Filter) -> match(words(Name), words(Filter)); +match(#share{} = Name, Filter) -> + match_share(Name, Filter); match([], []) -> true; match([H | T1], [H | T2]) -> @@ -87,12 +98,26 @@ match([_H1 | _], []) -> match([], [_H | _T2]) -> false. +-spec match_share(Name, Filter) -> boolean() when + Name :: share(), + Filter :: topic() | share(). +match_share(#share{topic = Name}, Filter) when is_binary(Filter) -> + %% only match real topic filter for normal topic filter. + match(words(Name), words(Filter)); +match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) -> + %% Matching real topic filter When subed same share group. + match(words(Name), words(Filter)); +match_share(#share{}, _) -> + %% Otherwise, non-matched. + false. + -spec match_any(Name, [Filter]) -> boolean() when Name :: topic() | words(), Filter :: topic() | words(). match_any(Topic, Filters) -> lists:any(fun(Filter) -> match(Topic, Filter) end, Filters). +%% TODO: validate share topic #share{} for emqx_trace.erl %% @doc Validate topic name or filter -spec validate(topic() | {name | filter, topic()}) -> true. validate(Topic) when is_binary(Topic) -> @@ -107,7 +132,7 @@ validate(_, <<>>) -> validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) -> %% MQTT-5.0 [MQTT-4.7.3-3] error(topic_too_long); -validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) -> +validate(filter, SharedFilter = <>) -> validate_share(SharedFilter); validate(filter, Filter) when is_binary(Filter) -> validate2(words(Filter)); @@ -139,12 +164,12 @@ validate3(<>) when C == $#; C == $+; C == 0 -> validate3(<<_/utf8, Rest/binary>>) -> validate3(Rest). -validate_share(<<"$share/", Rest/binary>>) when +validate_share(<>) when Rest =:= <<>> orelse Rest =:= <<"/">> -> %% MQTT-5.0 [MQTT-4.8.2-1] error(?SHARE_EMPTY_FILTER); -validate_share(<<"$share/", Rest/binary>>) -> +validate_share(<>) -> case binary:split(Rest, <<"/">>) of %% MQTT-5.0 [MQTT-4.8.2-1] [<<>>, _] -> @@ -156,7 +181,7 @@ validate_share(<<"$share/", Rest/binary>>) -> validate_share(ShareName, Filter) end. -validate_share(_, <<"$share/", _Rest/binary>>) -> +validate_share(_, <>) -> error(?SHARE_RECURSIVELY); validate_share(ShareName, Filter) -> case binary:match(ShareName, [<<"+">>, <<"#">>]) of @@ -185,7 +210,9 @@ bin('#') -> <<"#">>; bin(B) when is_binary(B) -> B; bin(L) when is_list(L) -> list_to_binary(L). --spec levels(topic()) -> pos_integer(). +-spec levels(topic() | share()) -> pos_integer(). +levels(#share{topic = Topic}) when is_binary(Topic) -> + levels(Topic); levels(Topic) when is_binary(Topic) -> length(tokens(Topic)). @@ -197,6 +224,8 @@ tokens(Topic) -> %% @doc Split Topic Path to Words -spec words(topic()) -> words(). +words(#share{topic = Topic}) when is_binary(Topic) -> + words(Topic); words(Topic) when is_binary(Topic) -> [word(W) || W <- tokens(Topic)]. @@ -237,26 +266,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) -> do_join(TopicAcc, [Word | Words]) -> do_join(<>, Words). --spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}. +-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}. parse(TopicFilter) when is_binary(TopicFilter) -> parse(TopicFilter, #{}); parse({TopicFilter, Options}) when is_binary(TopicFilter) -> parse(TopicFilter, Options). --spec parse(topic(), map()) -> {topic(), map()}. -parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) -> - error({invalid_topic_filter, TopicFilter}); -parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) -> - error({invalid_topic_filter, TopicFilter}); -parse(<<"$queue/", TopicFilter/binary>>, Options) -> - parse(TopicFilter, Options#{share => <<"$queue">>}); -parse(TopicFilter = <<"$share/", Rest/binary>>, Options) -> +-spec parse(topic() | share(), map()) -> {topic() | share(), map()}. +%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">> +%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>` +parse(#share{topic = Topic = <>}, _Options) -> + error({invalid_topic_filter, Topic}); +parse(#share{topic = Topic = <>}, _Options) -> + error({invalid_topic_filter, Topic}); +parse(<>, Options) -> + parse(#share{group = <>, topic = Topic}, Options); +parse(TopicFilter = <>, Options) -> case binary:split(Rest, <<"/">>) of [_Any] -> error({invalid_topic_filter, TopicFilter}); - [ShareName, Filter] -> - case binary:match(ShareName, [<<"+">>, <<"#">>]) of - nomatch -> parse(Filter, Options#{share => ShareName}); + %% `Group` could be `$share` or `$queue` + [Group, Topic] -> + case binary:match(Group, [<<"+">>, <<"#">>]) of + nomatch -> parse(#share{group = Group, topic = Topic}, Options); _ -> error({invalid_topic_filter, TopicFilter}) end end; @@ -267,5 +299,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) -> _ -> {Topic, Options#{is_exclusive => true}} end; -parse(TopicFilter, Options) -> +parse(TopicFilter, Options) when + ?IS_TOPIC(TopicFilter) +-> {TopicFilter, Options}. + +get_shared_real_topic(#share{topic = TopicFilter}) -> + TopicFilter; +get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) -> + TopicFilter. + +make_shared_record(Group, Topic) -> + #share{group = Group, topic = Topic}. + +maybe_format_share(#share{group = <>, topic = Topic}) -> + join([<>, Topic]); +maybe_format_share(#share{group = Group, topic = Topic}) -> + join([<>, Group, Topic]); +maybe_format_share(Topic) -> + join([Topic]). diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 504540cf6..dbd788c04 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -40,6 +40,10 @@ words/0 ]). +-export_type([ + share/0 +]). + -export_type([ socktype/0, sockstate/0, @@ -136,11 +140,14 @@ -type subid() :: binary() | atom(). --type group() :: binary() | undefined. +%% '_' for match spec +-type group() :: binary() | '_'. -type topic() :: binary(). -type word() :: '' | '+' | '#' | binary(). -type words() :: list(word()). +-type share() :: #share{}. + -type socktype() :: tcp | udp | ssl | proxy | atom(). -type sockstate() :: idle | running | blocked | closed. -type conninfo() :: #{ diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index a205f6fcd..da108ceef 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -299,7 +299,9 @@ t_nosub_pub(Config) when is_list(Config) -> ?assertEqual(1, emqx_metrics:val('messages.dropped')). t_shared_subscribe({init, Config}) -> - emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}), + emqx_broker:subscribe( + emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{} + ), ct:sleep(100), Config; t_shared_subscribe(Config) when is_list(Config) -> @@ -316,7 +318,7 @@ t_shared_subscribe(Config) when is_list(Config) -> end ); t_shared_subscribe({'end', _Config}) -> - emqx_broker:unsubscribe(<<"$share/group/topic">>). + emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)). t_shared_subscribe_2({init, Config}) -> Config; @@ -723,24 +725,6 @@ t_connack_auth_error(Config) when is_list(Config) -> ?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')), ok. -t_handle_in_empty_client_subscribe_hook({init, Config}) -> - Hook = {?MODULE, client_subscribe_delete_all_hook, []}, - ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100), - Config; -t_handle_in_empty_client_subscribe_hook({'end', _Config}) -> - emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}), - ok; -t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) -> - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - try - {ok, _, RCs} = emqtt:subscribe(C, <<"t">>), - ?assertEqual([?RC_UNSPECIFIED_ERROR], RCs), - ok - after - emqtt:disconnect(C) - end. - authenticate_deny(_Credentials, _Default) -> {stop, {error, bad_username_or_password}}. @@ -800,7 +784,3 @@ recv_msgs(Count, Msgs) -> after 100 -> Msgs end. - -client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) -> - EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter], - {stop, EmptyFilters}. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 8f6a2baaa..c6b4c0518 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -456,7 +456,7 @@ t_process_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], {[{TopicFilter, ?RC_SUCCESS}], _Channel} = - emqx_channel:process_subscribe(TopicFilters, #{}, channel()). + emqx_channel:process_subscribe(TopicFilters, channel()). t_process_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end), @@ -914,7 +914,13 @@ t_check_pub_alias(_) -> t_check_sub_authzs(_) -> emqx_config:put_zone_conf(default, [authorization, enable], true), TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, - [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()). + SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]), + CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]), + Channel = channel(), + ?assertEqual( + {ok, CheckedSubPkt, Channel}, + emqx_channel:check_sub_authzs(SubPkt, Channel) + ). t_enrich_connack_caps(_) -> ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]), @@ -1061,6 +1067,7 @@ clientinfo(InitProps) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, + is_bridge => false, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl index 6d065d521..0bfde981c 100644 --- a/apps/emqx/test/emqx_mountpoint_SUITE.erl +++ b/apps/emqx/test/emqx_mountpoint_SUITE.erl @@ -29,6 +29,7 @@ ). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -52,6 +53,27 @@ t_mount(_) -> mount(<<"device/1/">>, TopicFilters) ). +t_mount_share(_) -> + T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>), + TopicFilters = [T], + ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), + + %% should not mount share topic when make message. + Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), + + ?assertEqual( + TopicFilter, + mount(undefined, TopicFilter) + ), + ?assertEqual( + #share{group = <<"group">>, topic = <<"device/1/topic">>}, + mount(<<"device/1/">>, TopicFilter) + ), + ?assertEqual( + [{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}], + mount(<<"device/1/">>, TopicFilters) + ). + t_unmount(_) -> Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>), ?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)), @@ -61,6 +83,23 @@ t_unmount(_) -> ?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)), ?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)). +t_unmount_share(_) -> + {TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>), + MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>}, + + ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), + + %% should not unmount share topic when make message. + Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), + ?assertEqual( + TopicFilter, + unmount(undefined, TopicFilter) + ), + ?assertEqual( + #share{group = <<"group">>, topic = <<"topic">>}, + unmount(<<"device/1/">>, MountedTopicFilter) + ). + t_replvar(_) -> ?assertEqual(undefined, replvar(undefined, #{})), ?assertEqual( diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index 297ee7f7d..e97684b74 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -76,6 +76,8 @@ t_check_sub(_) -> ), ?assertEqual( {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true}) + emqx_mqtt_caps:check_sub( + ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts + ) ), emqx_config:put([zones], OldConf). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 4b4535cea..86887eff0 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -137,7 +137,8 @@ t_random_basic(Config) when is_list(Config) -> ClientId = <<"ClientId">>, Topic = <<"foo">>, Payload = <<"hello">>, - emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}), + Group = <<"group1">>, + emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}), MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload), %% wait for the subscription to show up ct:sleep(200), @@ -402,7 +403,7 @@ t_hash(Config) when is_list(Config) -> ok = ensure_config(hash_clientid, false), test_two_messages(hash_clientid). -t_hash_clinetid(Config) when is_list(Config) -> +t_hash_clientid(Config) when is_list(Config) -> ok = ensure_config(hash_clientid, false), test_two_messages(hash_clientid). @@ -528,14 +529,15 @@ last_message(ExpectedPayload, Pids, Timeout) -> t_dispatch(Config) when is_list(Config) -> ok = ensure_config(random), Topic = <<"foo">>, + Group = <<"group1">>, ?assertEqual( {error, no_subscribers}, - emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}}) + emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}}) ), - emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}), + emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}), ?assertEqual( {ok, 1}, - emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}}) + emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}}) ). t_uncovered_func(Config) when is_list(Config) -> @@ -991,37 +993,110 @@ t_session_kicked(Config) when is_list(Config) -> ?assertEqual([], collect_msgs(0)), ok. -%% FIXME: currently doesn't work -%% t_different_groups_same_topic({init, Config}) -> -%% TestName = atom_to_binary(?FUNCTION_NAME), -%% ClientId = <>, -%% {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), -%% {ok, _} = emqtt:connect(C), -%% [{client, C}, {clientid, ClientId} | Config]; -%% t_different_groups_same_topic({'end', Config}) -> -%% C = ?config(client, Config), -%% emqtt:stop(C), -%% ok; -%% t_different_groups_same_topic(Config) when is_list(Config) -> -%% C = ?config(client, Config), -%% ClientId = ?config(clientid, Config), -%% %% Subscribe and unsubscribe to both $queue and $shared topics -%% Topic = <<"t/1">>, -%% SharedTopic0 = <<"$share/aa/", Topic/binary>>, -%% SharedTopic1 = <<"$share/bb/", Topic/binary>>, -%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}), -%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}), +-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS), + ?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS})) +). -%% Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>), -%% emqx:publish(Message0), -%% ?assertMatch([ {publish, #{payload := <<"hi">>}} -%% , {publish, #{payload := <<"hi">>}} -%% ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}), +t_different_groups_same_topic({init, Config}) -> + TestName = atom_to_binary(?FUNCTION_NAME), + ClientId = <>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]; +t_different_groups_same_topic({'end', Config}) -> + C = ?config(client, Config), + emqtt:stop(C), + ok; +t_different_groups_same_topic(Config) when is_list(Config) -> + C = ?config(client, Config), + ClientId = ?config(clientid, Config), + %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic + GroupA = <<"aa">>, + GroupB = <<"bb">>, + Topic = <<"t/1">>, -%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0), -%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1), + SharedTopicGroupA = ?SHARE(GroupA, Topic), + ?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2), + SharedTopicGroupB = ?SHARE(GroupB, Topic), + ?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2), -%% ok. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(2, length(emqx_router:match_routes(Topic))) + end + ), + + Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>), + emqx:publish(Message0), + ?assertMatch( + [ + {publish, #{payload := <<"hi">>}}, + {publish, #{payload := <<"hi">>}} + ], + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} + ), + + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB), + + ok. + +t_different_groups_update_subopts({init, Config}) -> + TestName = atom_to_binary(?FUNCTION_NAME), + ClientId = <>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]; +t_different_groups_update_subopts({'end', Config}) -> + C = ?config(client, Config), + emqtt:stop(C), + ok; +t_different_groups_update_subopts(Config) when is_list(Config) -> + C = ?config(client, Config), + ClientId = ?config(clientid, Config), + %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic + Topic = <<"t/1">>, + GroupA = <<"aa">>, + GroupB = <<"bb">>, + SharedTopicGroupA = ?SHARE(GroupA, Topic), + SharedTopicGroupB = ?SHARE(GroupB, Topic), + + Fun = fun(Group, QoS) -> + ?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS), + ?assertMatch( + #{qos := QoS}, + emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic)) + ) + end, + + [Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]], + + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(2, length(emqx_router:match_routes(Topic))) + end + ), + + Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>), + emqx:publish(Message0), + ?assertMatch( + [ + {publish, #{payload := <<"hi">>}}, + {publish, #{payload := <<"hi">>}} + ], + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} + ), + + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB), + + ok. t_queue_subscription({init, Config}) -> TestName = atom_to_binary(?FUNCTION_NAME), @@ -1038,23 +1113,19 @@ t_queue_subscription({'end', Config}) -> t_queue_subscription(Config) when is_list(Config) -> C = ?config(client, Config), ClientId = ?config(clientid, Config), - %% Subscribe and unsubscribe to both $queue and $shared topics + %% Subscribe and unsubscribe to both $queue share and $share/ with same topic Topic = <<"t/1">>, QueueTopic = <<"$queue/", Topic/binary>>, SharedTopic = <<"$share/aa/", Topic/binary>>, - {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}), - {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}), - %% FIXME: we should actually see 2 routes, one for each group - %% ($queue and aa), but currently the latest subscription - %% overwrites the existing one. + ?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2), + ?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2), + ?retry( _Sleep0 = 100, _Attempts0 = 50, begin - ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), - %% FIXME: should ensure we have 2 subscriptions - [_] = emqx_router:lookup_routes(Topic) + ?assertEqual(2, length(emqx_router:match_routes(Topic))) end ), @@ -1063,37 +1134,29 @@ t_queue_subscription(Config) when is_list(Config) -> emqx:publish(Message0), ?assertMatch( [ + {publish, #{payload := <<"hi">>}}, {publish, #{payload := <<"hi">>}} - %% FIXME: should receive one message from each group - %% , {publish, #{payload := <<"hi">>}} ], - collect_msgs(5_000) + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} ), {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic), - %% FIXME: return code should be success instead of 17 ("no_subscription_existed") - {ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic), - %% FIXME: this should eventually be true, but currently we leak - %% the previous group subscription... - %% ?retry( - %% _Sleep0 = 100, - %% _Attempts0 = 50, - %% begin - %% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), - %% [] = emqx_router:lookup_routes(Topic) - %% end - %% ), + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(0, length(emqx_router:match_routes(Topic))) + end + ), ct:sleep(500), Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>), emqx:publish(Message1), - %% FIXME: we should *not* receive any messages... - %% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}), - %% This is from the leaked group... - ?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{ - routes => ets:tab2list(emqx_route) - }), + %% we should *not* receive any messages. + ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}), ok. diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index c49c93fb2..4761ea17d 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -238,11 +238,11 @@ long_topic() -> t_parse(_) -> ?assertError( {invalid_topic_filter, <<"$queue/t">>}, - parse(<<"$queue/t">>, #{share => <<"g">>}) + parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{}) ), ?assertError( {invalid_topic_filter, <<"$share/g/t">>}, - parse(<<"$share/g/t">>, #{share => <<"g">>}) + parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{}) ), ?assertError( {invalid_topic_filter, <<"$share/t">>}, @@ -254,8 +254,12 @@ t_parse(_) -> ), ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)), ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})), - ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)), - ?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)), + ?assertEqual( + {#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>) + ), + ?assertEqual( + {#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>) + ), %% The '$local' and '$fastlane' topics have been deprecated. ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)), ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)), diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index eb81c4b6e..0f21a2593 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -96,7 +96,7 @@ choose_ingress_pool_size( #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} ) -> case emqx_topic:parse(RemoteTopic) of - {_Filter, #{share := _Name}} -> + {#share{} = _Filter, _SubOpts} -> % NOTE: this is shared subscription, many workers may subscribe PoolSize; {_Filter, #{}} when PoolSize > 1 -> diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index b4358969d..64061de3d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -143,7 +143,7 @@ on_client_authorize(ClientInfo, Action, Topic, Result) -> Req = #{ clientinfo => clientinfo(ClientInfo), type => Type, - topic => Topic, + topic => emqx_topic:maybe_format_share(Topic), result => Bool }, case @@ -191,7 +191,7 @@ on_session_created(ClientInfo, _SessInfo) -> on_session_subscribed(ClientInfo, Topic, SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), - topic => Topic, + topic => emqx_topic:maybe_format_share(Topic), subopts => maps:with([qos, share, rh, rap, nl], SubOpts) }, cast('session.subscribed', Req). @@ -199,7 +199,7 @@ on_session_subscribed(ClientInfo, Topic, SubOpts) -> on_session_unsubscribed(ClientInfo, Topic, _SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), - topic => Topic + topic => emqx_topic:maybe_format_share(Topic) }, cast('session.unsubscribed', Req). @@ -413,7 +413,13 @@ enrich_header(Headers, Message) -> end. topicfilters(Tfs) when is_list(Tfs) -> - [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. + GetQos = fun(SubOpts) -> + maps:get(qos, SubOpts, 0) + end, + [ + #{name => emqx_topic:maybe_format_share(Topic), qos => GetQos(SubOpts)} + || {Topic, SubOpts} <- Tfs + ]. ntoa({0, 0, 0, 0, 0, 16#ffff, AB, CD}) -> list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256})); diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 2c9b5bb06..10462d210 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -547,6 +547,7 @@ subopts(SubOpts) -> rh => maps:get(rh, SubOpts, 0), rap => maps:get(rap, SubOpts, 0), nl => maps:get(nl, SubOpts, 0), + %% TOOD: FIXME for share-sub refactored share => maps:get(share, SubOpts, <<>>) }. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 6b4793b1c..a0436298e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -629,15 +629,8 @@ subscriptions(get, #{bindings := #{clientid := ClientID}}) -> {200, []}; {Node, Subs} -> Formatter = - fun({Topic, SubOpts}) -> - maps:merge( - #{ - node => Node, - clientid => ClientID, - topic => Topic - }, - maps:with([qos, nl, rap, rh], SubOpts) - ) + fun(_Sub = {Topic, SubOpts}) -> + emqx_mgmt_api_subscriptions:format(Node, {{Topic, ClientID}, SubOpts}) end, {200, lists:map(Formatter, Subs)} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index e6e8bb475..d10c9d068 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_router.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -38,8 +39,6 @@ format/2 ]). --define(SUBS_QTABLE, emqx_suboption). - -define(SUBS_QSCHEMA, [ {<<"clientid">>, binary}, {<<"topic">>, binary}, @@ -146,7 +145,7 @@ subscriptions(get, #{query_string := QString}) -> case maps:get(<<"node">>, QString, undefined) of undefined -> emqx_mgmt_api:cluster_query( - ?SUBS_QTABLE, + ?SUBOPTION, QString, ?SUBS_QSCHEMA, fun ?MODULE:qs2ms/2, @@ -157,7 +156,7 @@ subscriptions(get, #{query_string := QString}) -> {ok, Node1} -> emqx_mgmt_api:node_query( Node1, - ?SUBS_QTABLE, + ?SUBOPTION, QString, ?SUBS_QSCHEMA, fun ?MODULE:qs2ms/2, @@ -177,23 +176,16 @@ subscriptions(get, #{query_string := QString}) -> {200, Result} end. -format(WhichNode, {{Topic, _Subscriber}, Options}) -> +format(WhichNode, {{Topic, _Subscriber}, SubOpts}) -> maps:merge( #{ - topic => get_topic(Topic, Options), - clientid => maps:get(subid, Options, null), + topic => emqx_topic:maybe_format_share(Topic), + clientid => maps:get(subid, SubOpts, null), node => WhichNode }, - maps:with([qos, nl, rap, rh], Options) + maps:with([qos, nl, rap, rh], SubOpts) ). -get_topic(Topic, #{share := <<"$queue">> = Group}) -> - emqx_topic:join([Group, Topic]); -get_topic(Topic, #{share := Group}) -> - emqx_topic:join([<<"$share">>, Group, Topic]); -get_topic(Topic, _) -> - Topic. - %%-------------------------------------------------------------------- %% QueryString to MatchSpec %%-------------------------------------------------------------------- @@ -213,10 +205,18 @@ gen_match_spec([{Key, '=:=', Value} | More], MtchHead) -> update_ms(clientid, X, {{Topic, Pid}, Opts}) -> {{Topic, Pid}, Opts#{subid => X}}; -update_ms(topic, X, {{_Topic, Pid}, Opts}) -> +update_ms(topic, X, {{Topic, Pid}, Opts}) when + is_record(Topic, share) +-> + {{#share{group = '_', topic = X}, Pid}, Opts}; +update_ms(topic, X, {{Topic, Pid}, Opts}) when + is_binary(Topic) orelse Topic =:= '_' +-> {{X, Pid}, Opts}; -update_ms(share_group, X, {{Topic, Pid}, Opts}) -> - {{Topic, Pid}, Opts#{share => X}}; +update_ms(share_group, X, {{Topic, Pid}, Opts}) when + not is_record(Topic, share) +-> + {{#share{group = X, topic = Topic}, Pid}, Opts}; update_ms(qos, X, {{Topic, Pid}, Opts}) -> {{Topic, Pid}, Opts#{qos => X}}. @@ -227,5 +227,6 @@ fuzzy_filter_fun(Fuzzy) -> run_fuzzy_filter(_, []) -> true; -run_fuzzy_filter(E = {{Topic, _}, _}, [{topic, match, TopicFilter} | Fuzzy]) -> - emqx_topic:match(Topic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy). +run_fuzzy_filter(E = {{SubedTopic, _}, _}, [{topic, match, TopicFilter} | Fuzzy]) -> + {Filter, _SubOpts} = emqx_topic:parse(TopicFilter), + emqx_topic:match(SubedTopic, Filter) andalso run_fuzzy_filter(E, Fuzzy). diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index b6294ecbf..94bedd39f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -17,6 +17,7 @@ -module(emqx_mgmt_api_topics). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_router.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -101,10 +102,10 @@ fields(topic) -> %%%============================================================================================== %% parameters trans topics(get, #{query_string := Qs}) -> - do_list(generate_topic(Qs)). + do_list(Qs). topic(get, #{bindings := Bindings}) -> - lookup(generate_topic(Bindings)). + lookup(Bindings). %%%============================================================================================== %% api apply @@ -139,13 +140,6 @@ lookup(#{topic := Topic}) -> %%%============================================================================================== %% internal -generate_topic(Params = #{<<"topic">> := Topic}) -> - Params#{<<"topic">> => Topic}; -generate_topic(Params = #{topic := Topic}) -> - Params#{topic => Topic}; -generate_topic(Params) -> - Params. - -spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter(). qs2ms(_Tab, {Qs, _}) -> #{ @@ -160,9 +154,9 @@ gen_match_spec([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) -> gen_match_spec([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) -> gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]). -format(#route{topic = Topic, dest = {_, Node}}) -> - #{topic => Topic, node => Node}; -format(#route{topic = Topic, dest = Node}) -> +format(#route{topic = Topic, dest = {Group, Node}}) -> + #{topic => ?SHARE(Group, Topic), node => Node}; +format(#route{topic = Topic, dest = Node}) when is_atom(Node) -> #{topic => Topic, node => Node}. topic_param(In) -> diff --git a/apps/emqx_modules/src/emqx_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl index d91371f7e..619b53be4 100644 --- a/apps/emqx_modules/src/emqx_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -150,11 +150,12 @@ compile(Rules) -> Rules ). +%% FIXME: rewrite #share{} and return #share{}, not formated $share/group/topic match_and_rewrite(Topic, [], _) -> Topic; match_and_rewrite(Topic, [{Filter, MP, Dest} | Rules], Binds) -> case emqx_topic:match(Topic, Filter) of - true -> rewrite(Topic, MP, Dest, Binds); + true -> rewrite(emqx_topic:get_shared_real_topic(Topic), MP, Dest, Binds); false -> match_and_rewrite(Topic, Rules, Binds) end. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 4976e2400..49831a9e8 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -20,6 +20,7 @@ -include("emqx_retainer.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -export([start_link/0]). @@ -87,7 +88,7 @@ %% Hook API %%------------------------------------------------------------------------------ -spec on_session_subscribed(_, _, emqx_types:subopts(), _) -> any(). -on_session_subscribed(_, _, #{share := ShareName}, _) when ShareName =/= undefined -> +on_session_subscribed(_, #share{} = _Topic, _SubOpts, _) -> ok; on_session_subscribed(_, Topic, #{rh := Rh} = Opts, Context) -> IsNew = maps:get(is_new, Opts, true), diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 3ff588f48..ac9e077eb 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -190,7 +190,9 @@ on_session_subscribed(ClientInfo, Topic, SubOpts, Conf) -> apply_event( 'session.subscribed', fun() -> - eventmsg_sub_or_unsub('session.subscribed', ClientInfo, Topic, SubOpts) + eventmsg_sub_or_unsub( + 'session.subscribed', ClientInfo, emqx_topic:maybe_format_share(Topic), SubOpts + ) end, Conf ). @@ -199,7 +201,9 @@ on_session_unsubscribed(ClientInfo, Topic, SubOpts, Conf) -> apply_event( 'session.unsubscribed', fun() -> - eventmsg_sub_or_unsub('session.unsubscribed', ClientInfo, Topic, SubOpts) + eventmsg_sub_or_unsub( + 'session.unsubscribed', ClientInfo, emqx_topic:maybe_format_share(Topic), SubOpts + ) end, Conf ). diff --git a/changes/ce/fix-10976.en.md b/changes/ce/fix-10976.en.md new file mode 100644 index 000000000..87a7b442a --- /dev/null +++ b/changes/ce/fix-10976.en.md @@ -0,0 +1,2 @@ +Fix topic-filter overlapping handling in shared subscription. +In the previous implementation, the storage method for subscription options did not provide adequate support for shared subscriptions. This resulted in message routing failures and leakage of routing tables between nodes during the "subscribe-unsubscribe" process with specific order and topics. From 0ca725ff25841938afc3726c5b2cf7b4592ef9b1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 15 Aug 2023 11:22:55 +0800 Subject: [PATCH 026/111] fix: deprecated `share` in `subopts` --- apps/emqx/src/emqx_types.erl | 1 - apps/emqx_exhook/priv/protos/exhook.proto | 5 ++++- apps/emqx_exhook/src/emqx_exhook_handler.erl | 2 +- apps/emqx_exhook/test/props/prop_exhook_hooks.erl | 4 +--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index dbd788c04..1a4825736 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -214,7 +214,6 @@ rap := 0 | 1, nl := 0 | 1, qos := qos(), - share => binary(), atom() => term() }. -type reason_code() :: 0..16#FF. diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 928e9b20b..e5d7b3606 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -460,8 +460,11 @@ message SubOpts { // The QoS level uint32 qos = 1; + // deprecated + reserved 2; + reserved "share"; // The group name for shared subscription - string share = 2; + // string share = 2; // The Retain Handling option (MQTT v5.0) // diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 64061de3d..2bcb91b12 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -192,7 +192,7 @@ on_session_subscribed(ClientInfo, Topic, SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), topic => emqx_topic:maybe_format_share(Topic), - subopts => maps:with([qos, share, rh, rap, nl], SubOpts) + subopts => maps:with([qos, rh, rap, nl], SubOpts) }, cast('session.subscribed', Req). diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 10462d210..cf48fff80 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -546,9 +546,7 @@ subopts(SubOpts) -> qos => maps:get(qos, SubOpts, 0), rh => maps:get(rh, SubOpts, 0), rap => maps:get(rap, SubOpts, 0), - nl => maps:get(nl, SubOpts, 0), - %% TOOD: FIXME for share-sub refactored - share => maps:get(share, SubOpts, <<>>) + nl => maps:get(nl, SubOpts, 0) }. authresult_to_bool(AuthResult) -> From eaa5459509468a881a676334e199b86d0274eda2 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 8 Aug 2023 14:20:22 +0800 Subject: [PATCH 027/111] chore: bump apps vsn --- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 8a57249e9..79c34e36b 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, From 9732e31395a48bba236f13805468d12497ef2115 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 11 Oct 2023 11:23:41 +0800 Subject: [PATCH 028/111] chore: highlight breaking changes --- changes/ce/fix-10976.en.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changes/ce/fix-10976.en.md b/changes/ce/fix-10976.en.md index 87a7b442a..f2f15d4c8 100644 --- a/changes/ce/fix-10976.en.md +++ b/changes/ce/fix-10976.en.md @@ -1,2 +1,9 @@ Fix topic-filter overlapping handling in shared subscription. In the previous implementation, the storage method for subscription options did not provide adequate support for shared subscriptions. This resulted in message routing failures and leakage of routing tables between nodes during the "subscribe-unsubscribe" process with specific order and topics. + +## Breaking changes +* Hook callback `session.subscribed` and `client.subscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from `subopts`. +* Hook callback `session.unsubscribed` and `client.unsubscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. +* ExHook Proto changed. The `share` field in message `SubOpts` was deprecated. + ExHook Server will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from message `SubOpts`. +* `session.subscribed` and `session.unsubscribed` rule-engine events will have shared subscriptions in their full representation for `topic`, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. From 802a36c67020369e5d3a1020433bfa6303a14a5b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 25 Oct 2023 16:13:45 +0800 Subject: [PATCH 029/111] fix: find SubOpts by shared_record, not deliver topic --- apps/emqx/src/emqx_session.erl | 18 +++++++----------- apps/emqx/src/emqx_shared_sub.erl | 12 ++++++++---- apps/emqx/test/emqx_broker_SUITE.erl | 5 ++++- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index e79c30f4a..c31efb0a6 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -409,18 +409,14 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> [Msg | enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session)] end. -enrich_deliver( - ClientInfo, - {deliver, Topic, Msg = #message{headers = #{redispatch_to := {Group, Topic}}}}, - UpgradeQoS, - Session -) -> - %% Only QoS_1 and QoS_2 messages added `redispatch_to` header - %% For QoS 0 message, send it as regular dispatch - Deliver = {deliver, emqx_topic:make_shared_record(Group, Topic), Msg}, - enrich_deliver(ClientInfo, Deliver, UpgradeQoS, Session); enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> - SubOpts = ?IMPL(Session):get_subscription(Topic, Session), + SubOpts = + case Msg of + #message{headers = #{shared_record := SharedRecord}} -> + ?IMPL(Session):get_subscription(SharedRecord, Session); + _ -> + ?IMPL(Session):get_subscription(Topic, Session) + end, enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS). enrich_message( diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 84921be6b..c5ee9e7ab 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -141,14 +141,15 @@ record(Group, Topic, SubPid) -> dispatch(Group, Topic, Delivery) -> dispatch(Group, Topic, Delivery, _FailedSubs = #{}). -dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> - #message{from = ClientId, topic = SourceTopic} = Msg, +dispatch(Group, Topic, Delivery = #delivery{message = Msg0}, FailedSubs) -> + #message{from = ClientId, topic = SourceTopic} = Msg0, + Msg1 = with_shared_record(Msg0, Group, Topic), case pick(strategy(Group), ClientId, SourceTopic, Group, Topic, FailedSubs) of false -> {error, no_subscribers}; {Type, SubPid} -> - Msg1 = with_redispatch_to(Msg, Group, Topic), - case do_dispatch(SubPid, Group, Topic, Msg1, Type) of + Msg2 = with_redispatch_to(Msg1, Group, Topic), + case do_dispatch(SubPid, Group, Topic, Msg2, Type) of ok -> {ok, 1}; {error, Reason} -> @@ -239,6 +240,9 @@ with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) -> with_redispatch_to(Msg, Group, Topic) -> emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg). +with_shared_record(Msg, Group, Topic) -> + emqx_message:set_headers(#{shared_record => emqx_topic:make_shared_record(Group, Topic)}, Msg). + %% @hidden Redispatch is needed only for the messages with redispatch_to header added. is_redispatch_needed(#message{} = Msg) -> case get_redispatch_to(Msg) of diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index da108ceef..2bc6bd8ea 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -308,7 +308,10 @@ t_shared_subscribe(Config) when is_list(Config) -> emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assert( receive - {deliver, <<"topic">>, #message{payload = <<"hello">>}} -> + {deliver, <<"topic">>, #message{ + headers = #{shared_record := #share{group = <<"group">>, topic = <<"topic">>}}, + payload = <<"hello">> + }} -> true; Msg -> ct:pal("Msg: ~p", [Msg]), From afec6fa2f6c547c52479332df1d92703925a0e8d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 25 Oct 2023 17:12:28 +0800 Subject: [PATCH 030/111] fix: TopicFilter may modified by `client.subscribe` hook --- apps/emqx/src/emqx_channel.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 61b31c6e1..81e01e1bd 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -796,19 +796,19 @@ do_gen_reason_codes(Acc, [], []) -> lists:reverse(Acc); do_gen_reason_codes( Acc, - [{_TF, ?RC_SUCCESS} | RestCheckedTF], - [{_TF, NRC} | RestTFWithNRC] + [{_, ?RC_SUCCESS} | RestTF], + [{_, NRC} | RestWithNRC] ) -> %% will passing through `process_subscribe/2` %% use NRC to override IintialRC - do_gen_reason_codes([NRC | Acc], RestCheckedTF, RestTFWithNRC); + do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC); do_gen_reason_codes( Acc, - [{_TF, InitialRC} | RestChecked], - RestTFWithNRC + [{_, InitialRC} | Rest], + RestWithNRC ) -> %% InitialRC is not `RC_SUCCESS`, use it. - do_gen_reason_codes([InitialRC | Acc], RestChecked, RestTFWithNRC). + do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC). %%-------------------------------------------------------------------- %% Process Unsubscribe From 53383991d9a065b20c0331f1e7168c89ae7a0326 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 25 Oct 2023 17:25:23 +0800 Subject: [PATCH 031/111] fix: rewrite #share{} and return #share{}, not formated $share/group/topic --- apps/emqx_modules/src/emqx_rewrite.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/src/emqx_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl index 619b53be4..485c41f29 100644 --- a/apps/emqx_modules/src/emqx_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -150,15 +150,16 @@ compile(Rules) -> Rules ). -%% FIXME: rewrite #share{} and return #share{}, not formated $share/group/topic match_and_rewrite(Topic, [], _) -> Topic; match_and_rewrite(Topic, [{Filter, MP, Dest} | Rules], Binds) -> case emqx_topic:match(Topic, Filter) of - true -> rewrite(emqx_topic:get_shared_real_topic(Topic), MP, Dest, Binds); + true -> rewrite(Topic, MP, Dest, Binds); false -> match_and_rewrite(Topic, Rules, Binds) end. +rewrite(SharedRecord = #share{topic = Topic}, MP, Dest, Binds) -> + SharedRecord#share{topic = rewrite(Topic, MP, Dest, Binds)}; rewrite(Topic, MP, Dest, Binds) -> case re:run(Topic, MP, [{capture, all_but_first, list}]) of {match, Captured} -> From 9bb22507df4cc266b325b7f4c7a4554b6b9834c5 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 25 Oct 2023 16:09:11 +0800 Subject: [PATCH 032/111] fix: dont use transation on audit log --- apps/emqx_audit/include/emqx_audit.hrl | 1 - apps/emqx_audit/src/emqx_audit.erl | 120 ++++++++---------- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 29 +++++ 3 files changed, 84 insertions(+), 66 deletions(-) diff --git a/apps/emqx_audit/include/emqx_audit.hrl b/apps/emqx_audit/include/emqx_audit.hrl index 1b4349387..8304a9060 100644 --- a/apps/emqx_audit/include/emqx_audit.hrl +++ b/apps/emqx_audit/include/emqx_audit.hrl @@ -5,7 +5,6 @@ -define(AUDIT, emqx_audit). -record(?AUDIT, { - seq, %% basic info created_at, node, diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index 4477bbd8b..d00aa493b 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -14,6 +14,8 @@ -export([start_link/0]). -export([log/1, log/2]). +-export([dirty_clean_expired/1]). + %% gen_server callbacks -export([ init/1, @@ -26,12 +28,15 @@ ]). -define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]). --define(CLEAN_EXPIRED_MS, 60 * 1000). + +-ifdef(TEST). +-define(INTERVAL, 100). +-else. +-define(INTERVAL, 2500). +-endif. to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) -> #?AUDIT{ - created_at = erlang:system_time(microsecond), - node = node(), operation_id = <<"">>, operation_type = atom_to_binary(Cmd), args = Args, @@ -62,8 +67,6 @@ to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api duration_ms := DurationMs } = Log, #?AUDIT{ - created_at = erlang:system_time(microsecond), - node = node(), from = From, source = Source, source_ip = SourceIp, @@ -81,8 +84,6 @@ to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api }; to_audit(#{from := event, event := Event}) -> #?AUDIT{ - created_at = erlang:system_time(microsecond), - node = node(), from = event, source = <<"">>, source_ip = <<"">>, @@ -100,8 +101,6 @@ to_audit(#{from := event, event := Event}) -> }; to_audit(#{from := erlang_console, function := F, args := Args}) -> #?AUDIT{ - created_at = erlang:system_time(microsecond), - node = node(), from = erlang_console, source = <<"">>, source_ip = <<"">>, @@ -127,7 +126,12 @@ log(Level, Meta1) -> emqx_audit:log(Meta2). log(Log) -> - gen_server:cast(?MODULE, {write, to_audit(Log)}). + Audit0 = to_audit(Log), + Audit = Audit0#?AUDIT{ + node = node(), + created_at = erlang:system_time(microsecond) + }, + mria:dirty_write(?AUDIT, Audit). start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). @@ -140,27 +144,28 @@ init([]) -> {record_name, ?AUDIT}, {attributes, record_info(fields, ?AUDIT)} ]), - {ok, #{}, {continue, setup}}. + case mria_rlog:role() of + core -> {ok, #{}, {continue, setup}}; + _ -> {ok, #{}} + end. -handle_continue(setup, #{} = State) -> +handle_continue(setup, State) -> ok = mria:wait_for_tables([?AUDIT]), clean_expired(), - {noreply, State}. + Interval = clean_expired_interval(), + {noreply, State#{interval => Interval}, Interval}. -handle_call(_Request, _From, State = #{}) -> - {reply, ok, State}. +handle_call(_Request, _From, State = #{interval := Interval}) -> + {reply, ignore, State, Interval}. -handle_cast({write, Log}, State) -> - _ = write_log(Log), - {noreply, State#{}, ?CLEAN_EXPIRED_MS}; -handle_cast(_Request, State = #{}) -> - {noreply, State}. +handle_cast(_Request, State = #{interval := Interval}) -> + {noreply, State, Interval}. -handle_info(timeout, State = #{}) -> +handle_info(timeout, State = #{interval := Interval}) -> clean_expired(), - {noreply, State, hibernate}; -handle_info(_Info, State = #{}) -> - {noreply, State}. + {noreply, State, Interval}; +handle_info(_Info, State = #{interval := Interval}) -> + {noreply, State, Interval}. terminate(_Reason, _State = #{}) -> ok. @@ -172,50 +177,35 @@ code_change(_OldVsn, State = #{}, _Extra) -> %%% Internal functions %%%=================================================================== -write_log(Log) -> - case - mria:transaction( - ?COMMON_SHARD, - fun(L) -> - New = - case mnesia:last(?AUDIT) of - '$end_of_table' -> 1; - LastId -> LastId + 1 - end, - mnesia:write(L#?AUDIT{seq = New}) - end, - [Log] - ) - of - {atomic, ok} -> - ok; - Reason -> - ?SLOG(warning, #{ - msg => "write_audit_log_failed", - reason => Reason - }) - end. - clean_expired() -> MaxSize = max_size(), - LatestId = latest_id(), - Min = LatestId - MaxSize, - %% MS = ets:fun2ms(fun(#?AUDIT{seq = Seq}) when Seq =< Min -> true end), - MS = [{#?AUDIT{seq = '$1', _ = '_'}, [{'=<', '$1', Min}], [true]}], - NumDeleted = mnesia:ets(fun ets:select_delete/2, [?AUDIT, MS]), - ?SLOG(debug, #{ - msg => "clean_audit_log", - latest_id => LatestId, - min => Min, - deleted_number => NumDeleted - }), - ok. - -latest_id() -> - case mnesia:dirty_last(?AUDIT) of - '$end_of_table' -> 0; - Seq -> Seq + CurSize = mnesia:table_info(?AUDIT, size), + case CurSize - MaxSize of + DelCount when DelCount > 0 -> + mria:async_dirty( + ?COMMON_SHARD, + fun ?MODULE:dirty_clean_expired/1, + [DelCount] + ); + _ -> + ok end. +dirty_clean_expired(DelCount) -> + dirty_clean_expired(mnesia:dirty_first(?AUDIT), DelCount). + +dirty_clean_expired(_, DelCount) when DelCount =< 0 -> ok; +dirty_clean_expired('$end_of_table', _DelCount) -> + ok; +dirty_clean_expired(CurKey, DeleteCount) -> + mnesia:dirty_delete(?AUDIT, CurKey), + dirty_clean_expired(mnesia:dirty_next(?AUDIT, CurKey), DeleteCount - 1). + max_size() -> emqx_conf:get([log, audit, max_filter_size], 5000). + +%% Try to make the time interval of each node is different. +%% 2 * Interval ~ 3 * Interval (5000~7500) +clean_expired_interval() -> + Interval = ?INTERVAL, + Interval * 2 + erlang:phash2(node(), Interval). diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index 6fb860b6e..fe6ddfa96 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -121,6 +121,35 @@ t_cli(_Config) -> ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])), ok. +t_max_size(_Config) -> + {ok, _} = emqx:update_config([log, audit, max_filter_size], 1000), + SizeFun = + fun() -> + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Limit = "limit=1000", + {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader), + #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]), + erlang:length(Data) + end, + InitSize = SizeFun(), + lists:foreach( + fun(_) -> + ok = emqx_ctl:run_command(["conf", "show", "log"]) + end, + lists:duplicate(100, 1) + ), + timer:sleep(110), + Size1 = SizeFun(), + ?assert(Size1 - InitSize >= 100, {Size1, InitSize}), + {ok, _} = emqx:update_config([log, audit, max_filter_size], 10), + %% wait for clean_expired + timer:sleep(500), + ExpectSize = emqx:get_config([log, audit, max_filter_size]), + Size2 = SizeFun(), + ?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}), + ok. + t_kickout_clients_without_log(_) -> process_flag(trap_exit, true), AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), From 995948f0e84608dce7e4e1f6eb807b9e40772fb2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 27 Oct 2023 11:49:04 +0800 Subject: [PATCH 033/111] refactor: remove seq from audit record --- apps/emqx/include/emqx_trace.hrl | 1 - apps/emqx/include/logger.hrl | 12 +- apps/emqx/src/config/emqx_config_logger.erl | 24 +-- apps/emqx/src/emqx_trace/emqx_trace.erl | 2 +- apps/emqx_audit/src/emqx_audit.erl | 164 +++++++++++------- apps/emqx_audit/src/emqx_audit_api.erl | 4 +- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 3 +- apps/emqx_conf/src/emqx_conf_cli.erl | 5 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 10 +- .../src/emqx_dashboard_audit.erl | 9 +- apps/emqx_machine/src/emqx_machine_boot.erl | 4 - .../src/emqx_machine_terminator.erl | 7 +- rel/i18n/emqx_audit_api.hocon | 3 +- rel/i18n/emqx_conf_schema.hocon | 4 +- 14 files changed, 141 insertions(+), 111 deletions(-) diff --git a/apps/emqx/include/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl index 62028bcc0..3f9316727 100644 --- a/apps/emqx/include/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -32,6 +32,5 @@ -define(SHARD, ?COMMON_SHARD). -define(MAX_SIZE, 30). --define(OWN_KEYS, [level, filters, filter_default, handlers]). -endif. diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 67f125e5f..904fae0d6 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -40,7 +40,9 @@ end ). +-define(AUDIT_HANDLER, emqx_audit). -define(TRACE_FILTER, emqx_trace_filter). +-define(OWN_KEYS, [level, filters, filter_default, handlers]). -define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)). @@ -62,15 +64,15 @@ end). -define(AUDIT(_LevelFun_, _MetaFun_), begin - case emqx_config:get([log, audit], #{enable => false}) of - #{enable := false} -> + case logger_config:get(logger, ?AUDIT_HANDLER) of + {error, {not_found, _}} -> ok; - #{enable := true, level := _AllowLevel_} -> + {ok, Handler = #{level := _AllowLevel_}} -> _Level_ = _LevelFun_, case logger:compare_levels(_AllowLevel_, _Level_) of _R_ when _R_ == lt; _R_ == eq -> - emqx_audit:log(_Level_, _MetaFun_); - gt -> + emqx_audit:log(_Level_, _MetaFun_, Handler); + _ -> ok end end diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 87baef627..ce74db8f0 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -26,7 +26,6 @@ -include("logger.hrl"). -define(LOG, [log]). --define(AUDIT_HANDLER, emqx_audit). add_handler() -> ok = emqx_config_handler:add_handler(?LOG, ?MODULE), @@ -97,8 +96,11 @@ update_log_handlers(NewHandlers) -> ok = application:set_env(kernel, logger, NewHandlers), ok. +%% Don't remove audit log handler here, we need record this removed action into audit log file. +%% we will remove audit log handler after audit log is record in emqx_audit:log/3. +update_log_handler({removed, ?AUDIT_HANDLER}) -> + ok; update_log_handler({removed, Id}) -> - audit("audit_disabled", Id), log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]), logger:remove_handler(Id); update_log_handler({Action, {handler, Id, Mod, Conf}}) -> @@ -107,7 +109,6 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> _ = logger:remove_handler(Id), case logger:add_handler(Id, Mod, Conf) of ok -> - audit("audit_enabled", Id), ok; %% Don't crash here, otherwise the cluster rpc will retry the wrong handler forever. {error, Reason} -> @@ -118,23 +119,6 @@ update_log_handler({Action, {handler, Id, Mod, Conf}}) -> end, ok. --ifdef(EMQX_RELEASE_EDITION). - --if(?EMQX_RELEASE_EDITION == ee). -audit(Event, ?AUDIT_HANDLER) -> - emqx_audit:log(alert, #{event => Event, from => event}); -audit(_, _) -> - ok. --else. -audit(_, _) -> - ok. --endif. - --else. -audit(_, _) -> - ok. --endif. - id_for_log(console) -> "log.console"; id_for_log(Other) -> "log.file." ++ atom_to_list(Other). diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 99bbcc5f9..6588c99dc 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> ignore -> ignore; Log -> - case logger_config:get(ets:whereis(logger), Id) of + case logger_config:get(logger, Id) of {ok, #{module := Module} = HandlerConfig0} -> HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0), try diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index d00aa493b..ceaf22507 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -12,9 +12,9 @@ %% API -export([start_link/0]). --export([log/1, log/2]). +-export([log/3]). --export([dirty_clean_expired/1]). +-export([trans_clean_expired/2]). %% gen_server callbacks -export([ @@ -32,7 +32,7 @@ -ifdef(TEST). -define(INTERVAL, 100). -else. --define(INTERVAL, 2500). +-define(INTERVAL, 10000). -endif. to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) -> @@ -50,8 +50,6 @@ to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) -> http_method = <<"">>, http_request = <<"">> }; -to_audit(#{http_method := get}) -> - ok; to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api -> #{ source := Source, @@ -82,23 +80,6 @@ to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api duration_ms = DurationMs, args = <<"">> }; -to_audit(#{from := event, event := Event}) -> - #?AUDIT{ - from = event, - source = <<"">>, - source_ip = <<"">>, - %% operation info - operation_id = iolist_to_binary(Event), - operation_type = <<"">>, - operation_result = <<"">>, - failure = <<"">>, - %% request detail - http_status_code = <<"">>, - http_method = <<"">>, - http_request = <<"">>, - duration_ms = 0, - args = <<"">> - }; to_audit(#{from := erlang_console, function := F, args := Args}) -> #?AUDIT{ from = erlang_console, @@ -117,15 +98,22 @@ to_audit(#{from := erlang_console, function := F, args := Args}) -> args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args])) }. -log(_Level, undefined) -> +log(_Level, undefined, _Handler) -> ok; -log(Level, Meta1) -> +log(Level, Meta1, Handler) -> Meta2 = Meta1#{time => logger:timestamp(), level => Level}, - Filter = [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], - emqx_trace:log(Level, Filter, undefined, Meta2), - emqx_audit:log(Meta2). + log_to_file(Level, Meta2, Handler), + log_to_db(Meta2), + remove_handler_when_disabled(). -log(Log) -> +remove_handler_when_disabled() -> + case emqx_config:get([log, audit, enable], false) of + true -> ok; + false -> _ = logger:remove_handler(?AUDIT_HANDLER) + end, + ok. + +log_to_db(Log) -> Audit0 = to_audit(Log), Audit = Audit0#?AUDIT{ node = node(), @@ -144,68 +132,112 @@ init([]) -> {record_name, ?AUDIT}, {attributes, record_info(fields, ?AUDIT)} ]), - case mria_rlog:role() of - core -> {ok, #{}, {continue, setup}}; - _ -> {ok, #{}} - end. + {ok, #{}, {continue, setup}}. handle_continue(setup, State) -> ok = mria:wait_for_tables([?AUDIT]), - clean_expired(), - Interval = clean_expired_interval(), - {noreply, State#{interval => Interval}, Interval}. + NewState = State#{role => mria_rlog:role()}, + ?AUDIT(alert, #{ + cmd => emqx, + args => ["start"], + version => emqx_release:version(), + from => cli, + duration_ms => 0 + }), + {noreply, NewState, interval(NewState)}. -handle_call(_Request, _From, State = #{interval := Interval}) -> - {reply, ignore, State, Interval}. +handle_call(_Request, _From, State) -> + {reply, ignore, State, interval(State)}. -handle_cast(_Request, State = #{interval := Interval}) -> - {noreply, State, Interval}. +handle_cast(_Request, State) -> + {noreply, State, interval(State)}. -handle_info(timeout, State = #{interval := Interval}) -> - clean_expired(), - {noreply, State, Interval}; -handle_info(_Info, State = #{interval := Interval}) -> - {noreply, State, Interval}. +handle_info(timeout, State) -> + ExtraWait = clean_expired_logs(), + {noreply, State, interval(State) + ExtraWait}; +handle_info(_Info, State) -> + {noreply, State, interval(State)}. -terminate(_Reason, _State = #{}) -> +terminate(_Reason, _State) -> ok. -code_change(_OldVsn, State = #{}, _Extra) -> +code_change(_OldVsn, State, _Extra) -> {ok, State}. %%%=================================================================== %%% Internal functions %%%=================================================================== -clean_expired() -> +%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds. +clean_expired_logs() -> MaxSize = max_size(), + Oldest = mnesia:dirty_first(?AUDIT), CurSize = mnesia:table_info(?AUDIT, size), case CurSize - MaxSize of - DelCount when DelCount > 0 -> - mria:async_dirty( - ?COMMON_SHARD, - fun ?MODULE:dirty_clean_expired/1, - [DelCount] - ); + DelSize when DelSize > 0 -> + case + mria:transaction( + ?COMMON_SHARD, + fun ?MODULE:trans_clean_expired/2, + [Oldest, DelSize] + ) + of + {atomic, ok} -> + 0; + {aborted, Reason} -> + ?SLOG(error, #{ + msg => "clean_expired_audit_aborted", + reason => Reason, + delete_size => DelSize, + current_size => CurSize, + max_count => MaxSize + }), + 60000 + end; _ -> - ok + 0 end. -dirty_clean_expired(DelCount) -> - dirty_clean_expired(mnesia:dirty_first(?AUDIT), DelCount). +trans_clean_expired(Oldest, DelCount) -> + First = mnesia:first(?AUDIT), + %% Other node already clean from the oldest record. + %% ensure not delete twice, otherwise records that should not be deleted will be deleted. + case First =:= Oldest of + true -> do_clean_expired(First, DelCount); + false -> ok + end. -dirty_clean_expired(_, DelCount) when DelCount =< 0 -> ok; -dirty_clean_expired('$end_of_table', _DelCount) -> +do_clean_expired(_, DelSize) when DelSize =< 0 -> ok; +do_clean_expired('$end_of_table', _DelSize) -> ok; -dirty_clean_expired(CurKey, DeleteCount) -> - mnesia:dirty_delete(?AUDIT, CurKey), - dirty_clean_expired(mnesia:dirty_next(?AUDIT, CurKey), DeleteCount - 1). +do_clean_expired(CurKey, DeleteSize) -> + mnesia:delete(?AUDIT, CurKey, sticky_write), + do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1). max_size() -> emqx_conf:get([log, audit, max_filter_size], 5000). -%% Try to make the time interval of each node is different. -%% 2 * Interval ~ 3 * Interval (5000~7500) -clean_expired_interval() -> - Interval = ?INTERVAL, - Interval * 2 + erlang:phash2(node(), Interval). +interval(#{role := replicant}) -> hibernate; +interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL). + +log_to_file(Level, Meta, #{module := Module} = Handler) -> + Log = #{level => Level, meta => Meta, msg => undefined}, + Handler1 = maps:without(?OWN_KEYS, Handler), + try + erlang:apply(Module, log, [Log, Handler1]) + catch + C:R:S -> + case logger:remove_handler(?AUDIT_HANDLER) of + ok -> + logger:internal_log( + error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S} + ); + {error, {not_found, _}} -> + ok; + {error, Reason} -> + logger:internal_log( + error, + {removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S} + ) + end + end. diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl index aaa364464..81553dca0 100644 --- a/apps/emqx_audit/src/emqx_audit_api.erl +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -59,7 +59,7 @@ schema("/audit") -> desc => ?DESC(filter_node) })}, {from, - ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console, event]), #{ + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{ in => query, required => false, example => <<"dashboard">>, @@ -175,7 +175,7 @@ fields(audit) -> desc => "The node name to which the log is created" })}, {from, - ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console, event]), #{ + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{ desc => "The source type of the log" })}, {source, diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index fe6ddfa96..eebb8f770 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -49,6 +49,7 @@ init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]), ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT), emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), + ok = emqx_config_logger:refresh_config(), application:set_env(emqx, boot_modules, []), emqx_conf_cli:load(), Config. @@ -144,7 +145,7 @@ t_max_size(_Config) -> ?assert(Size1 - InitSize >= 100, {Size1, InitSize}), {ok, _} = emqx:update_config([log, audit, max_filter_size], 10), %% wait for clean_expired - timer:sleep(500), + timer:sleep(250), ExpectSize = emqx:get_config([log, audit, max_filter_size]), Size2 = SizeFun(), ?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}), diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index b47d1f961..a5082a419 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -40,7 +40,10 @@ load() -> emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), emqx_ctl:register_command(?CONF, {?MODULE, conf}, []), - emqx_ctl:register_command(?AUDIT_MOD, {?MODULE, audit}, [hidden]), + case emqx_release:edition() of + ee -> emqx_ctl:register_command(?AUDIT_MOD, {?MODULE, audit}, [hidden]); + ce -> ok + end, ok. unload() -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index cf4330e34..ab6204235 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -72,7 +72,7 @@ start_listeners(Listeners) -> base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, - log => fun emqx_dashboard_audit:log/2, + log => audit_log_fun(), security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], swagger_global_spec => GlobalSpec, dispatch => dispatch(), @@ -210,9 +210,17 @@ filter_false(K, V, S) -> [{K, V} | S]. listener_name(Protocol) -> list_to_atom(atom_to_list(Protocol) ++ ":dashboard"). +audit_log_fun() -> + case emqx_release:edition() of + ee -> fun emqx_dashboard_audit:log/2; + ce -> undefined + end. + -if(?EMQX_RELEASE_EDITION =/= ee). + %% dialyzer complains about the `unauthorized_role' clause... -dialyzer({no_match, [authorize/1, api_key_authorize/3]}). + -endif. authorize(Req) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index c2ef1a99f..4b51b2cb0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -37,10 +37,11 @@ log(#{code := Code, method := Method} = Meta, Req) -> ?AUDIT(level(Method, Code), log_meta(Meta, Req)). log_meta(Meta, Req) -> - #{operation_id := OperationId} = Meta, + #{operation_id := OperationId, method := Method} = Meta, case - lists:member(OperationId, ?HIGH_FREQUENCY_REQUESTS) andalso - ignore_high_frequency_request() + Method =:= get orelse + (lists:member(OperationId, ?HIGH_FREQUENCY_REQUESTS) andalso + ignore_high_frequency_request()) of true -> undefined; @@ -53,7 +54,7 @@ log_meta(Meta, Req) -> source_ip => source_ip(Req), operation_type => operation_type(Meta), %% method for http filter api. - http_method => maps:get(method, Meta), + http_method => Method, http_request => http_request(Meta), http_status_code => maps:get(code, Meta), operation_result => operation_result(Meta), diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index aa6180e23..026a82cbf 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -47,10 +47,6 @@ post_boot() -> ok = ensure_apps_started(), ok = print_vsn(), ok = start_autocluster(), - ?AUDIT(alert, #{ - event => "emqx_start", - from => event - }), ignore. -ifdef(TEST). diff --git a/apps/emqx_machine/src/emqx_machine_terminator.erl b/apps/emqx_machine/src/emqx_machine_terminator.erl index d43d5fea9..28603b9f6 100644 --- a/apps/emqx_machine/src/emqx_machine_terminator.erl +++ b/apps/emqx_machine/src/emqx_machine_terminator.erl @@ -68,8 +68,11 @@ graceful() -> %% @doc Shutdown the Erlang VM and wait indefinitely. graceful_wait() -> ?AUDIT(alert, #{ - event => "emqx_gracefully_stop", - from => event + cmd => emqx, + args => ["stop"], + version => emqx_release:version(), + from => cli, + duration_ms => element(1, erlang:statistics(wall_clock)) }), ok = graceful(), exit_loop(). diff --git a/rel/i18n/emqx_audit_api.hocon b/rel/i18n/emqx_audit_api.hocon index 37080838b..040c0009f 100644 --- a/rel/i18n/emqx_audit_api.hocon +++ b/rel/i18n/emqx_audit_api.hocon @@ -17,13 +17,12 @@ filter_from.desc: `rest_api`: API KEY request logs. `cli`: The emqx command line logs. `erlang_console`: The emqx remote_console run function logs. -`event`: Logs related to events such as emqx_start, emqx_gracefully_stop, audit_enabled, and audit_disabled.""" filter_source.desc: """"Filter logs based on source, Possible values are: The login username when logs are generated from the dashboard. The API Keys when logs are generated from the REST API. -empty string when logs are generated from CLI, Erlang console, or an event.""" +empty string when logs are generated from CLI, Erlang console.""" filter_source_ip.desc: "Filter logs based on source ip when logs are generated from dashboard and REST API." diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index ff2c3109a..2497244a5 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -726,7 +726,9 @@ audit_handler_level.label: """Log Level""" audit_log_max_filter_limit.desc: -"""Store the latest N log entries in a database for allow `/audit` HTTP API to filter and retrieval of log data.""" +"""Store the latest N log entries in a database for allow `/audit` HTTP API to filter and retrieval of log data. +The interval for purging redundant log records is maintained within a range of 10~20 seconds. +""" audit_log_max_filter_limit.label: """Max Filter Limit""" From 22223dc536586b721d1a967424957ca2fc22c218 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 27 Oct 2023 16:11:30 +0800 Subject: [PATCH 034/111] fix: return 400 when audit log is disabled --- apps/emqx_audit/src/emqx_audit_api.erl | 48 ++++++++++------- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 52 +++++++++++++++++-- rel/i18n/emqx_audit_api.hocon | 2 +- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl index 81553dca0..a7fd8f4ad 100644 --- a/apps/emqx_audit/src/emqx_audit_api.erl +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -35,6 +35,7 @@ {<<"gte_duration_ms">>, timestamp}, {<<"lte_duration_ms">>, timestamp} ]). +-define(DISABLE_MSG, <<"Audit is disabled">>). namespace() -> "audit". @@ -151,7 +152,11 @@ schema("/audit") -> emqx_dashboard_swagger:schema_with_example( array(?REF(audit_list)), audit_log_list_example() - ) + ), + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], + ?DISABLE_MSG + ) } } }. @@ -232,23 +237,30 @@ fields(http_request) -> ]. audit(get, #{query_string := QueryString}) -> - case - emqx_mgmt_api:node_query( - node(), - ?AUDIT, - QueryString, - ?AUDIT_QS_SCHEMA, - fun ?MODULE:qs2ms/2, - fun ?MODULE:format/1 - ) - of - {error, page_limit_invalid} -> - {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; - {error, Node, Error} -> - Message = list_to_binary(io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error])), - {500, #{code => <<"NODE_DOWN">>, message => Message}}; - Result -> - {200, Result} + case emqx_config:get([log, audit, enable], false) of + false -> + {400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}}; + true -> + case + emqx_mgmt_api:node_query( + node(), + ?AUDIT, + QueryString, + ?AUDIT_QS_SCHEMA, + fun ?MODULE:qs2ms/2, + fun ?MODULE:format/1 + ) + of + {error, page_limit_invalid} -> + {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + {error, Node, Error} -> + Message = list_to_binary( + io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error]) + ), + {500, #{code => <<"NODE_DOWN">>, message => Message}}; + Result -> + {200, Result} + end end. qs2ms(_Tab, {Qs, _}) -> diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index eebb8f770..a402efe31 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -19,6 +19,21 @@ -include_lib("eunit/include/eunit.hrl"). +all() -> + [ + {group, enabled}, + {group, disabled} + ]. + +groups() -> + [ + {enabled, [sequence], common_tests() -- [t_disabled]}, + {disabled, [sequence], [t_disabled]} + ]. + +common_tests() -> + emqx_common_test_helpers:all(?MODULE). + -define(CONF_DEFAULT, #{ node => #{ @@ -40,9 +55,6 @@ } }). -all() -> - emqx_common_test_helpers:all(?MODULE). - init_per_suite(Config) -> _ = application:load(emqx_conf), emqx_config:erase_all(), @@ -90,6 +102,40 @@ t_http_api(_) -> ), ok. +t_disabled(_) -> + Enable = [log, audit, enable], + ?assertEqual(true, emqx:get_config(Enable)), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + Size1 = mnesia:table_info(emqx_audit, size), + + {ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"), + Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100), + NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false), + {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs), + ?assertMatch( + {error, _}, + emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader) + ), + + Size2 = mnesia:table_info(emqx_audit, size), + %% Record the audit disable action, so the size + 1 + ?assertEqual(Size1 + 1, Size2), + + {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111), + {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones), + ?assertMatch(#{<<"max_topic_levels">> := 111}, Res), + Size3 = mnesia:table_info(emqx_audit, size), + %% Don't record mqtt update request. + ?assertEqual(Size2, Size3), + %% enabled again + {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1), + Size4 = mnesia:table_info(emqx_audit, size), + ?assertEqual(Size3 + 1, Size4), + ok. + t_cli(_Config) -> ok = emqx_ctl:run_command(["conf", "show", "log"]), AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), diff --git a/rel/i18n/emqx_audit_api.hocon b/rel/i18n/emqx_audit_api.hocon index 040c0009f..40741310e 100644 --- a/rel/i18n/emqx_audit_api.hocon +++ b/rel/i18n/emqx_audit_api.hocon @@ -16,7 +16,7 @@ filter_from.desc: `dashboard`: Dashboard request logs, requiring the use of a jwt_token. `rest_api`: API KEY request logs. `cli`: The emqx command line logs. -`erlang_console`: The emqx remote_console run function logs. +`erlang_console`: The emqx remote_console run function logs.""" filter_source.desc: """"Filter logs based on source, Possible values are: From 3b5cc912e7c5ed4e11d2e02d684ab5dcf4f8a726 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 27 Oct 2023 16:15:23 +0800 Subject: [PATCH 035/111] fix: add `redispatch_to` header to all msgs when deliver shared-sub - to find correct SubOpts for shared-sub dispatch - use previous key `redispatch_to` to ensure rolling upgrade compatibility --- apps/emqx/src/emqx_session.erl | 4 ++-- apps/emqx/src/emqx_shared_sub.erl | 29 +++++++++++----------------- apps/emqx/test/emqx_broker_SUITE.erl | 2 +- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index c31efb0a6..f5157aaf3 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -412,8 +412,8 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> SubOpts = case Msg of - #message{headers = #{shared_record := SharedRecord}} -> - ?IMPL(Session):get_subscription(SharedRecord, Session); + #message{headers = #{redispatch_to := {Group, T}}} -> + ?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session); _ -> ?IMPL(Session):get_subscription(Topic, Session) end, diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index c5ee9e7ab..691ab7497 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -141,15 +141,14 @@ record(Group, Topic, SubPid) -> dispatch(Group, Topic, Delivery) -> dispatch(Group, Topic, Delivery, _FailedSubs = #{}). -dispatch(Group, Topic, Delivery = #delivery{message = Msg0}, FailedSubs) -> - #message{from = ClientId, topic = SourceTopic} = Msg0, - Msg1 = with_shared_record(Msg0, Group, Topic), +dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> + #message{from = ClientId, topic = SourceTopic} = Msg, case pick(strategy(Group), ClientId, SourceTopic, Group, Topic, FailedSubs) of false -> {error, no_subscribers}; {Type, SubPid} -> - Msg2 = with_redispatch_to(Msg1, Group, Topic), - case do_dispatch(SubPid, Group, Topic, Msg2, Type) of + Msg1 = with_redispatch_to(Msg, Group, Topic), + case do_dispatch(SubPid, Group, Topic, Msg1, Type) of ok -> {ok, 1}; {error, Reason} -> @@ -235,22 +234,16 @@ without_group_ack(Msg) -> get_group_ack(Msg) -> emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK). -with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) -> - Msg; +%% always add `redispatch_to` header to the message +%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1 with_redispatch_to(Msg, Group, Topic) -> emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg). -with_shared_record(Msg, Group, Topic) -> - emqx_message:set_headers(#{shared_record => emqx_topic:make_shared_record(Group, Topic)}, Msg). - -%% @hidden Redispatch is needed only for the messages with redispatch_to header added. -is_redispatch_needed(#message{} = Msg) -> - case get_redispatch_to(Msg) of - ?REDISPATCH_TO(_, _) -> - true; - _ -> - false - end. +%% @hidden Redispatch is needed only for the messages which not QOS_0 +is_redispatch_needed(#message{qos = ?QOS_0}) -> + false; +is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) -> + true. %% @doc Redispatch shared deliveries to other members in the group. redispatch(Messages0) -> diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index 2bc6bd8ea..18d9f9651 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -309,7 +309,7 @@ t_shared_subscribe(Config) when is_list(Config) -> ?assert( receive {deliver, <<"topic">>, #message{ - headers = #{shared_record := #share{group = <<"group">>, topic = <<"topic">>}}, + headers = #{redispatch_to := {<<"group">>, <<"topic">>}}, payload = <<"hello">> }} -> true; From 4ba34f8f3eaa0756296e5fbd1f11074d47fb8fc5 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 25 Oct 2023 12:02:40 +0800 Subject: [PATCH 036/111] chore(rbac): fix CI errors & update change --- .../src/emqx_dashboard_rbac.erl | 4 - apps/emqx_management/src/emqx_mgmt_auth.erl | 18 +++- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 88 ++++++++++++++++++- changes/ee/feat-11811.en.md | 5 ++ rel/i18n/emqx_mgmt_api_key_schema.hocon | 7 +- 5 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 changes/ee/feat-11811.en.md diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index cd38540dd..7b8ffef02 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -59,12 +59,8 @@ valid_role(Type, Role) -> %% =================================================================== check_rbac(?ROLE_SUPERUSER, _, _, _) -> true; -%%check_rbac(?ROLE_API_SUPERUSER, _, _, _) -> -%% true; check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) -> true; -%%check_rbac(?ROLE_API_VIEWER, <<"GET">>, _, _) -> -%% true; check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>, _) -> true; check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>, _) -> diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index bdb5d97fa..fa48ccfea 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -38,7 +38,7 @@ -export([authorize/4]). -export([post_config_update/5]). --export([backup_tables/0]). +-export([backup_tables/0, validate_mnesia_backup/1]). %% Internal exports (RPC) -export([ @@ -82,6 +82,22 @@ mnesia(boot) -> backup_tables() -> [?APP]. +validate_mnesia_backup({schema, _Tab, CreateList} = Schema) -> + case emqx_mgmt_data_backup:default_validate_mnesia_backup(Schema) of + ok -> + ok; + _ -> + case proplists:get_value(attributes, CreateList) of + %% Since v5.4.0 the `desc` has changed to `extra` + [name, api_key, api_secret_hash, enable, desc, expired_at, created_at] -> + ok; + Fields -> + {error, {unknow_fields, Fields}} + end + end; +validate_mnesia_backup(_Other) -> + ok. + post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) -> #{bootstrap_file := File} = NewConf, case init_bootstrap_file(File) of diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 8243b18ff..bebf8e338 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -39,7 +39,7 @@ groups() -> [ {parallel, [parallel], [t_create, t_update, t_delete, t_authorize, t_create_unexpired_app]}, {parallel, [parallel], ?EE_CASES}, - {sequence, [], [t_bootstrap_file, t_create_failed]} + {sequence, [], [t_bootstrap_file, t_bootstrap_file_with_role, t_create_failed]} ]. init_per_suite(Config) -> @@ -86,6 +86,92 @@ t_bootstrap_file(_) -> update_file(<<>>), ok. +-if(?EMQX_RELEASE_EDITION == ee). +t_bootstrap_file_with_role(_) -> + Search = fun(Name) -> + lists:search( + fun(#{api_key := AppName}) -> + AppName =:= Name + end, + emqx_mgmt_auth:list() + ) + end, + + Bin = <<"role-1:role-1:viewer\nrole-2:role-2:administrator\nrole-3:role-3">>, + File = "./bootstrap_api_keys.txt", + ok = file:write_file(File, Bin), + update_file(File), + + ?assertMatch( + {value, #{api_key := <<"role-1">>, role := <<"viewer">>}}, + Search(<<"role-1">>) + ), + + ?assertMatch( + {value, #{api_key := <<"role-2">>, role := <<"administrator">>}}, + Search(<<"role-2">>) + ), + + ?assertMatch( + {value, #{api_key := <<"role-3">>, role := <<"administrator">>}}, + Search(<<"role-3">>) + ), + + %% bad role + BadBin = <<"role-4:secret-11:bad\n">>, + ok = file:write_file(File, BadBin), + update_file(File), + ?assertEqual( + false, + Search(<<"role-4">>) + ), + ok. +-else. +t_bootstrap_file_with_role(_) -> + Search = fun(Name) -> + lists:search( + fun(#{api_key := AppName}) -> + AppName =:= Name + end, + emqx_mgmt_auth:list() + ) + end, + + Bin = <<"role-1:role-1:administrator\nrole-2:role-2">>, + File = "./bootstrap_api_keys.txt", + ok = file:write_file(File, Bin), + update_file(File), + + ?assertMatch( + {value, #{api_key := <<"role-1">>, role := <<"administrator">>}}, + Search(<<"role-1">>) + ), + + ?assertMatch( + {value, #{api_key := <<"role-2">>, role := <<"administrator">>}}, + Search(<<"role-2">>) + ), + + %% only administrator + OtherRoleBin = <<"role-3:role-3:viewer\n">>, + ok = file:write_file(File, OtherRoleBin), + update_file(File), + ?assertEqual( + false, + Search(<<"role-3">>) + ), + + %% bad role + BadBin = <<"role-4:secret-11:bad\n">>, + ok = file:write_file(File, BadBin), + update_file(File), + ?assertEqual( + false, + Search(<<"role-4">>) + ), + ok. +-endif. + auth_authorize(Path, Key, Secret) -> FakePath = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri("/fake")), FakeReq = #{method => <<"GET">>, path => FakePath}, diff --git a/changes/ee/feat-11811.en.md b/changes/ee/feat-11811.en.md new file mode 100644 index 000000000..91b9e90aa --- /dev/null +++ b/changes/ee/feat-11811.en.md @@ -0,0 +1,5 @@ +Improve the format for the REST API key bootstrap file to support initialize key with a role. + +The new form is:`api_key:api_secret:role`. + +`role` is optional and its default value is `administrator`. diff --git a/rel/i18n/emqx_mgmt_api_key_schema.hocon b/rel/i18n/emqx_mgmt_api_key_schema.hocon index 811ab8a98..e72682747 100644 --- a/rel/i18n/emqx_mgmt_api_key_schema.hocon +++ b/rel/i18n/emqx_mgmt_api_key_schema.hocon @@ -9,8 +9,11 @@ api_key.label: bootstrap_file.desc: """The bootstrap file provides API keys for EMQX. EMQX will load these keys on startup to authorize API requests. -It contains key-value pairs in the format:`api_key:api_secret`. -Each line specifies an API key and its associated secret.""" +It contains colon-separated values in the format: `api_key:api_secret:role`. +Each line specifies an API key and its associated secret, and the role of this key. +The 'role' part should be the pre-defined access scope group name, +for example, `administrator` or `viewer`. +The 'role' is introduced in 5.4, to be backward compatible, if it is missing, the key is implicitly granted `administrator` role.""" bootstrap_file.label: """Initialize api_key file.""" From d563121284d63fd0ff79873223ec6308a09ee607 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 27 Oct 2023 17:54:14 +0800 Subject: [PATCH 037/111] refactor: move ?REDISPATCH_TO macro to emqx_mqtt.hrl --- apps/emqx/include/emqx_mqtt.hrl | 2 ++ apps/emqx/src/emqx_session.erl | 2 +- apps/emqx/src/emqx_shared_sub.erl | 1 - apps/emqx/test/emqx_broker_SUITE.erl | 2 +- apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl | 3 ++- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 93c70a6e1..53fed0f9d 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -675,6 +675,8 @@ end). -define(QUEUE, "$queue"). -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). +-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}). + -define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty). -define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty). -define(SHARE_RECURSIVELY, '$share_cannot_be_used_as_real_topic_filter'). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index f5157aaf3..147b0b35c 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -412,7 +412,7 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> SubOpts = case Msg of - #message{headers = #{redispatch_to := {Group, T}}} -> + #message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} -> ?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session); _ -> ?IMPL(Session):get_subscription(Topic, Session) diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 691ab7497..89a785590 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -95,7 +95,6 @@ -define(ACK, shared_sub_ack). -define(NACK(Reason), {shared_sub_nack, Reason}). -define(NO_ACK, no_ack). --define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}). -define(SUBSCRIBER_DOWN, noproc). -type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()). diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index 18d9f9651..b416f1730 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -309,7 +309,7 @@ t_shared_subscribe(Config) when is_list(Config) -> ?assert( receive {deliver, <<"topic">>, #message{ - headers = #{redispatch_to := {<<"group">>, <<"topic">>}}, + headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)}, payload = <<"hello">> }} -> true; diff --git a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl index f8fe49ca8..fee112d9a 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_events_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -42,7 +43,7 @@ t_printable_maps(_) -> peerhost => {127, 0, 0, 1}, peername => {{127, 0, 0, 1}, 9980}, sockname => {{127, 0, 0, 1}, 1883}, - redispatch_to => {<<"group">>, <<"sub/topic/+">>}, + redispatch_to => ?REDISPATCH_TO(<<"group">>, <<"sub/topic/+">>), shared_dispatch_ack => {self(), ref} }, Converted = emqx_rule_events:printable_maps(Headers), From 0634ff61c0570564f7113a042b194c5ca258256b Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 27 Oct 2023 17:27:38 +0800 Subject: [PATCH 038/111] fix: dialyzer warning --- apps/emqx/include/logger.hrl | 14 ++++++++++++++ apps/emqx_audit/src/emqx_audit.erl | 10 ++++++---- apps/emqx_audit/test/emqx_audit_api_SUITE.erl | 12 +++++++----- apps/emqx_conf/src/emqx_conf_cli.erl | 2 ++ apps/emqx_dashboard/src/emqx_dashboard.erl | 2 ++ rel/i18n/emqx_conf_schema.hocon | 3 +-- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 904fae0d6..a40f9dc9c 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -63,6 +63,10 @@ ) end). +-ifdef(EMQX_RELEASE_EDITION). + +-if(?EMQX_RELEASE_EDITION == ee). + -define(AUDIT(_LevelFun_, _MetaFun_), begin case logger_config:get(logger, ?AUDIT_HANDLER) of {error, {not_found, _}} -> @@ -78,6 +82,16 @@ end). end end). +-else. +%% Only for compile pass, ce edition will not call it +-define(AUDIT(_L_, _M_), _ = {_L_, _M_}). +-endif. + +-else. +%% Only for compile pass, ce edition will not call it +-define(AUDIT(_L_, _M_), _ = {_L_, _M_}). +-endif. + %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl index ceaf22507..98f4a70e8 100644 --- a/apps/emqx_audit/src/emqx_audit.erl +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -108,10 +108,12 @@ log(Level, Meta1, Handler) -> remove_handler_when_disabled() -> case emqx_config:get([log, audit, enable], false) of - true -> ok; - false -> _ = logger:remove_handler(?AUDIT_HANDLER) - end, - ok. + true -> + ok; + false -> + _ = logger:remove_handler(?AUDIT_HANDLER), + ok + end. log_to_db(Log) -> Audit0 = to_audit(Log), diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl index a402efe31..50b39d240 100644 --- a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -21,14 +21,12 @@ all() -> [ - {group, enabled}, - {group, disabled} + {group, audit, [sequence]} ]. groups() -> [ - {enabled, [sequence], common_tests() -- [t_disabled]}, - {disabled, [sequence], [t_disabled]} + {audit, [sequence], common_tests()} ]. common_tests() -> @@ -114,6 +112,8 @@ t_disabled(_) -> Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100), NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false), {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs), + {ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"), + ?assertEqual(NewLogs, GetLog1), ?assertMatch( {error, _}, emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader) @@ -132,6 +132,8 @@ t_disabled(_) -> ?assertEqual(Size2, Size3), %% enabled again {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1), + {ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"), + ?assertEqual(Logs1, GetLog2), Size4 = mnesia:table_info(emqx_audit, size), ?assertEqual(Size3 + 1, Size4), ok. @@ -243,4 +245,4 @@ kickout_clients() -> {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]), - ?assertMatch(#{<<"meta">> := #{<<"count">> := 0}}, ClientsResponse2). + ?assertMatch(#{<<"data">> := []}, ClientsResponse2). diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index a5082a419..fc00c7dc9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -37,6 +37,8 @@ -define(AUDIT_MOD, audit). -define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited"). +-dialyzer({no_match, [load/0]}). + load() -> emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), emqx_ctl:register_command(?CONF, {?MODULE, conf}, []), diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index ab6204235..96f81ca84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -210,6 +210,8 @@ filter_false(K, V, S) -> [{K, V} | S]. listener_name(Protocol) -> list_to_atom(atom_to_list(Protocol) ++ ":dashboard"). +-dialyzer({no_match, [audit_log_fun/0]}). + audit_log_fun() -> case emqx_release:edition() of ee -> fun emqx_dashboard_audit:log/2; diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 2497244a5..64b96541e 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -727,8 +727,7 @@ audit_handler_level.label: audit_log_max_filter_limit.desc: """Store the latest N log entries in a database for allow `/audit` HTTP API to filter and retrieval of log data. -The interval for purging redundant log records is maintained within a range of 10~20 seconds. -""" +The interval for purging redundant log records is maintained within a range of 10~20 seconds.""" audit_log_max_filter_limit.label: """Max Filter Limit""" From 814e22feb3038db995456447801529b7e5817a4c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 30 Oct 2023 00:04:27 +0800 Subject: [PATCH 039/111] fix: topics/subscripton mgmt api searching --- apps/emqx/src/emqx_topic.erl | 7 ++- .../src/emqx_mgmt_api_subscriptions.erl | 55 +++++++++++-------- .../src/emqx_mgmt_api_topics.erl | 11 +++- 3 files changed, 46 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 20dfd4316..76c6ef34e 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -83,6 +83,8 @@ match(Name, Filter) when is_binary(Name), is_binary(Filter) -> match(words(Name), words(Filter)); match(#share{} = Name, Filter) -> match_share(Name, Filter); +match(Name, #share{} = Filter) -> + match_share(Name, Filter); match([], []) -> true; match([H | T1], [H | T2]) -> @@ -109,7 +111,10 @@ match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = F match(words(Name), words(Filter)); match_share(#share{}, _) -> %% Otherwise, non-matched. - false. + false; +match_share(Name, #share{topic = Filter}) when is_binary(Name) -> + %% Only match real topic filter for normal topic_filter/topic_name. + match(Name, Filter). -spec match_any(Name, [Filter]) -> boolean() when Name :: topic() | words(), diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index d10c9d068..39ffb5972 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -142,31 +142,25 @@ parameters() -> subscriptions(get, #{query_string := QString}) -> Response = - case maps:get(<<"node">>, QString, undefined) of - undefined -> - emqx_mgmt_api:cluster_query( - ?SUBOPTION, - QString, - ?SUBS_QSCHEMA, - fun ?MODULE:qs2ms/2, - fun ?MODULE:format/2 - ); - Node0 -> - case emqx_utils:safe_to_existing_atom(Node0) of - {ok, Node1} -> - emqx_mgmt_api:node_query( - Node1, - ?SUBOPTION, - QString, - ?SUBS_QSCHEMA, - fun ?MODULE:qs2ms/2, - fun ?MODULE:format/2 - ); - {error, _} -> - {error, Node0, {badrpc, <<"invalid node">>}} + try + begin + case maps:get(<<"match_topic">>, QString, undefined) of + undefined -> + do_subscriptions_query(QString); + MatchTopic -> + case emqx_topic:parse(MatchTopic) of + {#share{}, _} -> {error, invalid_match_topic}; + _ -> do_subscriptions_query(QString) + end end + end + catch + error:{invalid_topic_filter, _} -> + {error, invalid_match_topic} end, case Response of + {error, invalid_match_topic} -> + {400, #{code => <<"INVALID_PARAMETER">>, message => <<"match_topic_invalid">>}}; {error, page_limit_invalid} -> {400, #{code => <<"INVALID_PARAMETER">>, message => <<"page_limit_invalid">>}}; {error, Node, {badrpc, R}} -> @@ -176,6 +170,20 @@ subscriptions(get, #{query_string := QString}) -> {200, Result} end. +do_subscriptions_query(QString) -> + Args = [?SUBOPTION, QString, ?SUBS_QSCHEMA, fun ?MODULE:qs2ms/2, fun ?MODULE:format/2], + case maps:get(<<"node">>, QString, undefined) of + undefined -> + erlang:apply(fun emqx_mgmt_api:cluster_query/5, Args); + Node0 -> + case emqx_utils:safe_to_existing_atom(Node0) of + {ok, Node1} -> + erlang:apply(fun emqx_mgmt_api:node_query/6, [Node1 | Args]); + {error, _} -> + {error, Node0, {badrpc, <<"invalid node">>}} + end + end. + format(WhichNode, {{Topic, _Subscriber}, SubOpts}) -> maps:merge( #{ @@ -228,5 +236,4 @@ fuzzy_filter_fun(Fuzzy) -> run_fuzzy_filter(_, []) -> true; run_fuzzy_filter(E = {{SubedTopic, _}, _}, [{topic, match, TopicFilter} | Fuzzy]) -> - {Filter, _SubOpts} = emqx_topic:parse(TopicFilter), - emqx_topic:match(SubedTopic, Filter) andalso run_fuzzy_filter(E, Fuzzy). + emqx_topic:match(SubedTopic, TopicFilter) andalso run_fuzzy_filter(E, Fuzzy). diff --git a/apps/emqx_management/src/emqx_mgmt_api_topics.erl b/apps/emqx_management/src/emqx_mgmt_api_topics.erl index 94bedd39f..31c70573f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_topics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_topics.erl @@ -149,8 +149,15 @@ qs2ms(_Tab, {Qs, _}) -> gen_match_spec([], Res) -> Res; -gen_match_spec([{topic, '=:=', T} | Qs], [{{route, _, N}, [], ['$_']}]) -> - gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]); +gen_match_spec([{topic, '=:=', T0} | Qs], [{{route, _, Node}, [], ['$_']}]) when is_atom(Node) -> + {T, D} = + case emqx_topic:parse(T0) of + {#share{group = Group, topic = Topic}, _SubOpts} -> + {Topic, {Group, Node}}; + {T1, _SubOpts} -> + {T1, Node} + end, + gen_match_spec(Qs, [{{route, T, D}, [], ['$_']}]); gen_match_spec([{node, '=:=', N} | Qs], [{{route, T, _}, [], ['$_']}]) -> gen_match_spec(Qs, [{{route, T, N}, [], ['$_']}]). From e9de7316b62354ba8c2c6b98baa2249c3665bcb9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 30 Oct 2023 14:41:57 +0800 Subject: [PATCH 040/111] test: shared-sub topics/subscription api --- .../test/emqx_mgmt_api_subscription_SUITE.erl | 50 +++++++++++++++++++ .../test/emqx_mgmt_api_topics_SUITE.erl | 32 ++++++++++++ 2 files changed, 82 insertions(+) diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index a23d70f2f..9ca3bb876 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -155,6 +155,56 @@ t_list_with_internal_subscription(_Config) -> ), ok. +t_list_with_shared_sub(_Config) -> + Client = proplists:get_value(client, _Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS = [ + {"clientid", ?CLIENTID}, + {"match_topic", "t/#"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := ?CLIENTID}, #{<<"clientid">> := ?CLIENTID}]}, + request_json(get, QS, Headers) + ), + + ok. + +t_list_with_invalid_match_topic(_Config) -> + Client = proplists:get_value(client, _Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS = [ + {"clientid", ?CLIENTID}, + {"match_topic", "$share/g1/t/1"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + {error, + {{_, 400, _}, _, #{ + <<"message">> := <<"match_topic_invalid">>, + <<"code">> := <<"INVALID_PARAMETER">> + }}}, + begin + {error, {R, _H, Body}} = emqx_mgmt_api_test_util:request_api( + get, path(), uri_string:compose_query(QS), Headers, [], #{return_all => true} + ), + {error, {R, _H, emqx_utils_json:decode(Body, [return_maps])}} + end + ), + ok. + request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index a2f546267..854f1133b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -123,3 +123,35 @@ t_percent_topics(_Config) -> ), ok = emqtt:stop(Client). + +t_shared_topics(_Configs) -> + Node = atom_to_binary(node(), utf8), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, Client} = emqtt:start_link(#{ + username => <<"routes_username">>, clientid => <<"routes_cid">> + }), + {ok, _} = emqtt:connect(Client), + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + %% exact match with shared topic + Path = emqx_mgmt_api_test_util:api_path(["topics"]), + QS = uri_string:compose_query([ + {"topic", Topic}, + {"node", atom_to_list(node())} + ]), + Headers = emqx_mgmt_api_test_util:auth_header_(), + {ok, MatchResponse1} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), + MatchData = emqx_utils_json:decode(MatchResponse1, [return_maps]), + ?assertMatch( + #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100}, + maps:get(<<"meta">>, MatchData) + ), + ?assertMatch( + [#{<<"topic">> := Topic, <<"node">> := Node}], + maps:get(<<"data">>, MatchData) + ), + + ok = emqtt:stop(Client). From 3a09fdc495058bef64335698c1de5184c24a2cad Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 30 Oct 2023 16:40:56 +0800 Subject: [PATCH 041/111] refactor: check match topic before do subscriptions query --- .../src/emqx_mgmt_api_subscriptions.erl | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 39ffb5972..ca0a7a625 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -142,21 +142,9 @@ parameters() -> subscriptions(get, #{query_string := QString}) -> Response = - try - begin - case maps:get(<<"match_topic">>, QString, undefined) of - undefined -> - do_subscriptions_query(QString); - MatchTopic -> - case emqx_topic:parse(MatchTopic) of - {#share{}, _} -> {error, invalid_match_topic}; - _ -> do_subscriptions_query(QString) - end - end - end - catch - error:{invalid_topic_filter, _} -> - {error, invalid_match_topic} + case check_match_topic(QString) of + ok -> do_subscriptions_query(QString); + {error, _} = Err -> Err end, case Response of {error, invalid_match_topic} -> @@ -170,6 +158,31 @@ subscriptions(get, #{query_string := QString}) -> {200, Result} end. +format(WhichNode, {{Topic, _Subscriber}, SubOpts}) -> + maps:merge( + #{ + topic => emqx_topic:maybe_format_share(Topic), + clientid => maps:get(subid, SubOpts, null), + node => WhichNode + }, + maps:with([qos, nl, rap, rh], SubOpts) + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +check_match_topic(#{<<"match_topic">> := MatchTopic}) -> + try emqx_topic:parse(MatchTopic) of + {#share{}, _} -> {error, invalid_match_topic}; + _ -> ok + catch + error:{invalid_topic_filter, _} -> + {error, invalid_match_topic} + end; +check_match_topic(_) -> + ok. + do_subscriptions_query(QString) -> Args = [?SUBOPTION, QString, ?SUBS_QSCHEMA, fun ?MODULE:qs2ms/2, fun ?MODULE:format/2], case maps:get(<<"node">>, QString, undefined) of @@ -184,16 +197,6 @@ do_subscriptions_query(QString) -> end end. -format(WhichNode, {{Topic, _Subscriber}, SubOpts}) -> - maps:merge( - #{ - topic => emqx_topic:maybe_format_share(Topic), - clientid => maps:get(subid, SubOpts, null), - node => WhichNode - }, - maps:with([qos, nl, rap, rh], SubOpts) - ). - %%-------------------------------------------------------------------- %% QueryString to MatchSpec %%-------------------------------------------------------------------- From 3ec811e828c1ce8c5ad2b814c544d952b2da401b Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 30 Oct 2023 21:45:09 +0800 Subject: [PATCH 042/111] feat(gbt32960): Port the GBT32960 gateway from v4 --- apps/emqx_gateway/src/emqx_gateway_api.erl | 52 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 3 +- apps/emqx_gateway_gbt32960/BSL.txt | 94 ++ apps/emqx_gateway_gbt32960/README.md | 24 + .../doc/Data_Exchange_Guide_CN.md | 741 +++++++++++++++ .../include/emqx_gbt32960.hrl | 75 ++ apps/emqx_gateway_gbt32960/rebar.config | 6 + .../src/emqx_gateway_gbt32960.app.src | 10 + .../src/emqx_gateway_gbt32960.erl | 97 ++ .../src/emqx_gbt32960_channel.erl | 862 ++++++++++++++++++ .../src/emqx_gbt32960_frame.erl | 802 ++++++++++++++++ .../src/emqx_gbt32960_schema.erl | 55 ++ apps/emqx_machine/priv/reboot_lists.eterm | 1 + mix.exs | 1 + rel/i18n/emqx_gateway_api.hocon | 2 +- rel/i18n/emqx_gbt32960_schema.hocon | 12 + 16 files changed, 2830 insertions(+), 7 deletions(-) create mode 100644 apps/emqx_gateway_gbt32960/BSL.txt create mode 100644 apps/emqx_gateway_gbt32960/README.md create mode 100644 apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md create mode 100644 apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl create mode 100644 apps/emqx_gateway_gbt32960/rebar.config create mode 100644 apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src create mode 100644 apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl create mode 100644 apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl create mode 100644 apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl create mode 100644 apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl create mode 100644 rel/i18n/emqx_gbt32960_schema.hocon diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 61f29059f..b628b47e3 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -161,12 +161,13 @@ gateway_enable(put, #{bindings := #{name := Name, enable := Enable}}) -> return_http_error(404, <<"NOT FOUND">>) end. --spec gw_name(binary()) -> stomp | coap | lwm2m | mqttsn | exproto | no_return(). +-spec gw_name(binary()) -> stomp | coap | lwm2m | mqttsn | exproto | gbt32960 | no_return(). gw_name(<<"stomp">>) -> stomp; gw_name(<<"coap">>) -> coap; gw_name(<<"lwm2m">>) -> lwm2m; gw_name(<<"mqttsn">>) -> mqttsn; gw_name(<<"exproto">>) -> exproto; +gw_name(<<"gbt32960">>) -> gbt32960; gw_name(_Else) -> throw(not_found). %%-------------------------------------------------------------------- @@ -390,7 +391,8 @@ fields(Gw) when Gw == mqttsn; Gw == coap; Gw == lwm2m; - Gw == exproto + Gw == exproto; + Gw == gbt32960 -> [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++ convert_listener_struct(emqx_gateway_schema:gateway_schema(Gw)); @@ -399,7 +401,8 @@ fields(Gw) when Gw == update_mqttsn; Gw == update_coap; Gw == update_lwm2m; - Gw == update_exproto + Gw == update_exproto; + Gw == update_gbt32960 -> "update_" ++ GwStr = atom_to_list(Gw), Gw1 = list_to_existing_atom(GwStr), @@ -458,7 +461,8 @@ schema_load_or_update_gateways_conf() -> ref(?MODULE, update_mqttsn), ref(?MODULE, update_coap), ref(?MODULE, update_lwm2m), - ref(?MODULE, update_exproto) + ref(?MODULE, update_exproto), + ref(?MODULE, update_gbt32960) ]), examples_update_gateway_confs() ). @@ -470,7 +474,8 @@ schema_gateways_conf() -> ref(?MODULE, mqttsn), ref(?MODULE, coap), ref(?MODULE, lwm2m), - ref(?MODULE, exproto) + ref(?MODULE, exproto), + ref(?MODULE, gbt32960) ]), examples_gateway_confs() ). @@ -756,6 +761,30 @@ examples_gateway_confs() -> } ] } + }, + gbt32960_gateway => + #{ + summary => <<"A simple GBT32960 gateway config">>, + value => + #{ + enable => true, + name => <<"gbt32960">>, + enable_stats => true, + mountpoint => <<"gbt32960/${clientid}">>, + retry_interval => <<"8s">>, + max_retry_times => 3, + message_queue_len => 10, + listeners => + [ + #{ + type => <<"tcp">>, + name => <<"default">>, + bind => <<"7325">>, + max_connections => 1024000, + max_conn_rate => 1000 + } + ] + } } }. @@ -854,5 +883,18 @@ examples_update_gateway_confs() -> handler => #{address => <<"http://127.0.0.1:9001">>} } + }, + gbt32960_gateway => + #{ + summary => <<"A simple GBT32960 gateway config">>, + value => + #{ + enable => true, + enable_stats => true, + mountpoint => <<"gbt32960/${clientid}">>, + retry_interval => <<"8s">>, + max_retry_times => 3, + message_queue_len => 10 + } } }. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index e58e552e2..2107d3133 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -341,7 +341,8 @@ gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); gateway_schema(coap) -> emqx_coap_schema:fields(coap); gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m); -gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto). +gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto); +gateway_schema(gbt32960) -> emqx_gbt32960_schema:fields(gbt32960). %%-------------------------------------------------------------------- %% helpers diff --git a/apps/emqx_gateway_gbt32960/BSL.txt b/apps/emqx_gateway_gbt32960/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_gateway_gbt32960/README.md b/apps/emqx_gateway_gbt32960/README.md new file mode 100644 index 000000000..779e7004c --- /dev/null +++ b/apps/emqx_gateway_gbt32960/README.md @@ -0,0 +1,24 @@ +# emqx_gbt32960 + +The GBT32960 Gateway is based on the GBT32960 specification. + +## Quick Start + +In EMQX 5.0, GBT32960 gateway can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: + +```properties +gateway.gbt32960 { + + mountpoint = "gbt32960/${clientid}" + + listeners.tcp.default { + bind = 7325 + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. diff --git a/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md new file mode 100644 index 000000000..e528f982a --- /dev/null +++ b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md @@ -0,0 +1,741 @@ +# emqx-gbt32960 + +该文档定义了 Plugins **emqx_gbt32960** 和 **EMQX** 之间数据交换的格式 + +约定: +- Payload 采用 Json 格式进行组装 +- Json Key 采用大驼峰格式命名 + +# Upstream +数据流向: Terminal -> emqx_gbt32960 -> EMQX + +## 车辆登入 +Topic: gbt32960/${vin}/upstream/vlogin + +```json +{ + "Cmd": 1, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "ICCID": "12345678901234567890", + "Id": "C", + "Length": 1, + "Num": 1, + "Seq": 1, + "Time": { + "Day": 29, + "Hour": 12, + "Minute": 19, + "Month": 12, + "Second": 20, + "Year": 12 + } + } +} +``` + +其中 + +| 字段 | 类型 | 描述 | +| --------- | ------- | ------------------------------------------------------------ | +| `Cmd` | Integer | 命令单元； `1` 表示车辆登入 | +| `Encrypt` | Integer | 数据单元加密方式，`1` 表示不加密，`2` 数据经过 RSA 加密，`3` 数据经过 ASE128 算法加密；`254` 表示异常；`255` 表示无效；其他预留 | +| `Vin` | String | 唯一识别码，即车辆 VIN 码 | +| `Data` | Object | 数据单元， JSON 对象格式。 | + +车辆登入的数据单元格式为 + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------------------------------------ | +| `Time` | Object | 数据采集时间，按年，月，日，时，分，秒，格式见示例。 | +| `Seq` | Integer | 登入流水号 | +| `ICCID` | String | 长度为20的字符串，SIM 卡的 ICCID 号 | +| `Num` | Integer | 可充电储能子系统数，有效值 0 ~ 250 | +| `Length` | Integer | 可充电储能系统编码长度，有效值 0 ~ 50 | +| `Id` | String | 可充电储能系统编码，长度为 "子系统数" 与 "编码长度" 值的乘积 | + +## 车辆登出 + +Topic: gbt32960/${vin}/upstream/vlogout + +车辆登出的 `Cmd` 值为 4，其余字段含义与登入相同： + +```json +{ + "Cmd": 4, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Seq": 1, + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +## 实时信息上报 + +Topic: gbt32960/${vin}/upstream/info + +> 不同信息类型上报，格式上只有 Infos 里面的对象属性不同，通过 `Type` 进行区分 +> Infos 为数组，代表车载终端每次报文可以上报多个信息 + +### 整车数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "AcceleratorPedal": 90, + "BrakePedal": 0, + "Charging": 1, + "Current": 15000, + "DC": 1, + "Gear": 5, + "Mileage": 999999, + "Mode": 1, + "Resistance": 6000, + "SOC": 50, + "Speed": 2000, + "Status": 1, + "Type": "Vehicle", + "Voltage": 5000 + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +其中，整车信息字段含义如下： + +| 字段 | 类型 | 描述 | +| ------------ | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，`Vehicle` 表示该结构为整车信息 | +| `Status` | Integer | 车辆状态，`1` 表示启动状态；`2` 表示熄火；`3` 表示其状态；`254` 表示异常；`255` 表示无效 | +| `Charging` | Integer | 充电状态，`1` 表示停车充电；`2` 行驶充电；`3` 未充电状态；`4` 充电完成；`254` 表示异常；`255` 表示无效 | +| `Mode` | Integer | 运行模式，`1` 表示纯电；`2` 混动；`3` 燃油；`254` 表示异常；`255` 表示无效 | +| `Speed` | Integer | 车速，有效值 （ 0~ 2200，表示 0 km/h ~ 220 km/h），单位 0.1 km/h | +| `Mileage` | Integer | 累计里程，有效值 0 ~9,999,999（表示 0 km ~ 999,999.9 km），单位 0.1 km | +| `Voltage` | Integer | 总电压，有效值范围 0 ~10000（表示 0 V ~ 1000 V）单位 0.1 V | +| `Current` | Integer | 总电流，有效值 0 ~ 20000 （偏移量 1000，表示 -1000 A ~ +1000 A，单位 0.1 A | +| `SOC` | Integer | SOC，有效值 0 ~ 100（表示 0% ~ 100%） | +| `DC` | Integer | DC，`1` 工作；`2` 断开；`254` 表示异常；`255` 表示无效 | +| `Gear` | Integer | 档位，参考原协议的 表 A.1，此值为其转换为整数的值 | +| `Resistance` | Integer | 绝缘电阻，有效范围 0 ~ 60000（表示 0 k欧姆 ~ 60000 k欧姆） | + +### 驱动电机数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Motors": [ + { + "CtrlTemp": 125, + "DCBusCurrent": 31203, + "InputVoltage": 30012, + "MotorTemp": 125, + "No": 1, + "Rotating": 30000, + "Status": 1, + "Torque": 25000 + }, + { + "CtrlTemp": 125, + "DCBusCurrent": 30200, + "InputVoltage": 32000, + "MotorTemp": 145, + "No": 2, + "Rotating": 30200, + "Status": 1, + "Torque": 25300 + } + ], + "Number": 2, + "Type": "DriveMotor" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，驱动电机数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------ | +| `Type` | String | 数据类型，此处为 `DriveMotor` | +| `Number` | Integer | 驱动电机个数，有效值 1~253 | +| `Motors` | Array | 驱动电机数据列表 | + +驱动电机数据字段为： + +| 字段 | 类型 | 描述 | +| -------------- | -------- | ------------------------------------------------------------ | +| `No` | Integer | 驱动电机序号，有效值 1~253 | +| `Status` | Integer | 驱动电机状态，`1` 表示耗电；`2`发电；`3` 关闭状态；`4` 准备状态；`254` 表示异常；`255` 表示无效 | +| `CtrlTemp` | Integer | 驱动电机控制器温度，有效值 0~250（数值偏移 40°C，表示 -40°C ~ +210°C）单位 °C | +| `Rotating` | Interger | 驱动电机转速，有效值 0~65531（数值偏移 20000表示 -20000 r/min ~ 45531 r/min）单位 1 r/min | +| `Torque` | Integer | 驱动电机转矩，有效值 0~65531（数据偏移量 20000，表示 - 2000 N·m ~ 4553.1 N·m）单位 0.1 N·m | +| `MotorTemp` | Integer | 驱动电机温度，有效值 0~250（数据偏移量 40 °C，表示 -40°C ~ +210°C）单位 1°C | +| `InputVoltage` | Integer | 电机控制器输入电压，有效值 0~60000（表示 0V ~ 6000V）单位 0.1 V | +| `DCBusCurrent` | Interger | 电机控制器直流母线电流，有效值 0~20000（数值偏移 1000A，表示 -1000A ~ +1000 A）单位 0.1 A | + +### 燃料电池数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CellCurrent": 12000, + "CellVoltage": 10000, + "DCStatus": 1, + "FuelConsumption": 45000, + "H_ConcSensorCode": 11, + "H_MaxConc": 35000, + "H_MaxPress": 500, + "H_MaxTemp": 12500, + "H_PressSensorCode": 12, + "H_TempProbeCode": 10, + "ProbeNum": 2, + "ProbeTemps": [120, 121], + "Type": "FuelCell" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，燃料电池数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `FuleCell` | +| `CellVoltage` | Integer | 燃料电池电压，有效值范围 0~20000（表示 0V ~ 2000V）单位 0.1 V | +| `CellCurrent` | Integer | 燃料电池电流，有效值范围 0~20000（表示 0A~ +2000A）单位 0.1 A | +| `FuelConsumption` | Integer | 燃料消耗率，有效值范围 0~60000（表示 0kg/100km ~ 600 kg/100km) 单位 0.01 kg/100km | +| `ProbeNum` | Integer | 燃料电池探针总数，有效值范围 0~65531 | +| `ProbeTemps` | Array | 燃料电池每探针温度值 | +| `H_MaxTemp` | Integer | 氢系统最高温度，有效值 0~2400（偏移量40°C，表示 -40°C~200°C）单位 0.1 °C | +| `H_TempProbeCode` | Integer | 氢系统最高温度探针代号，有效值 1~252 | +| `H_MaxConc` | Integer | 氢气最高浓度，有效值 0~60000（表示 0mg/kg ~ 50000 mg/kg）单位 1mg/kg | +| `H_ConcSensorCode` | Integer | 氢气最高浓度传感器代号，有效值 1~252 | +| `H_MaxPress` | Integer | 氢气最高压力，有效值 0~1000（表示 0 MPa ~ 100 MPa）最小单位 0.1 MPa | +| `H_PressSensorCode` | Integer | 氢气最高压力传感器代号，有效值 1~252 | +| `DCStatus` | Integer | 高压 DC/DC状态，`1` 表示工作；`2`断开 | + +### 发动机数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CrankshaftSpeed": 2000, + "FuelConsumption": 200, + "Status": 1, + "Type": "Engine" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，发动机数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ----------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Engine` | +| `Status` | Integer | 发动机状态，`1` 表示启动；`2` 关闭 | +| `CrankshaftSpeed` | Integer | 曲轴转速，有效值 0~60000（表示 0r/min~60000r/min）单位 1r/min | +| `FuelConsumption` | Integer | 燃料消耗率，有效范围 0~60000（表示 0L/100km~600L/100km）单位 0.01 L/100km | + + + +### 车辆位置数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Latitude": 100, + "Longitude": 10, + "Status": 0, + "Type": "Location" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，车辆位置数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ----------- | ------- | ----------------------------------------------------- | +| `Type` | String | 数据类型，此处为 `Location` | +| `Status` | Integer | 定位状态，见原协议表15，此处为所有比特位的整型值 | +| `Longitude` | Integer | 经度，以度为单位的纬度值乘以 10^6，精确到百万分之一度 | +| `Latitude` | Integer | 纬度，以度为单位的纬度值乘以 10^6，精确到百万分之一度 | + + + +### 极值数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "MaxBatteryVoltage": 7500, + "MaxTemp": 120, + "MaxTempProbeNo": 12, + "MaxTempSubsysNo": 14, + "MaxVoltageBatteryCode": 10, + "MaxVoltageBatterySubsysNo": 12, + "MinBatteryVoltage": 2000, + "MinTemp": 40, + "MinTempProbeNo": 13, + "MinTempSubsysNo": 15, + "MinVoltageBatteryCode": 11, + "MinVoltageBatterySubsysNo": 13, + "Type": "Extreme" + } + ], + "Time": { + "Day": 30, + "Hour": 12, + "Minute": 22, + "Month": 5, + "Second": 59, + "Year": 17 + } + } +} +``` + +其中，极值数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| --------------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Extreme` | +| `MaxVoltageBatterySubsysNo` | Integer | 最高电压电池子系统号，有效值 1~250 | +| `MaxVoltageBatteryCode` | Integer | 最高电压电池单体代号，有效值 1~250 | +| `MaxBatteryVoltage` | Integer | 电池单体电压最高值，有效值 0~15000（表示 0V~15V）单位 0.001V | +| `MinVoltageBatterySubsysNo` | Integer | 最低电压电池子系统号，有效值 1~250 | +| `MinVoltageBatteryCode` | Integer | 最低电压电池单体代号，有效值 1~250 | +| `MinBatteryVoltage` | Integer | 电池单体电压最低值，有效值 0~15000（表示 0V~15V）单位 0.001V | +| `MaxTempSubsysNo` | Integer | 最高温度子系统号，有效值 1~250 | +| `MaxTempProbeNo` | Integer | 最高温度探针序号，有效值 1~250 | +| `MaxTemp` | Integer | 最高温度值，有效值范围 0~250（偏移量40，表示 -40°C~+210°C） | +| `MinTempSubsysNo` | Integer | 最低温度子系统号，有效值 1~250 | +| `MinTempProbeNo` | Integer | 最低温度探针序号，有效值 1~250 | +| `MinTemp` | Integer | 最低温度值，有效值范围 0~250（偏移量40，表示 -40°C~+210°C） | + + + +### 报警数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "FaultChargeableDeviceNum": 1, + "FaultChargeableDeviceList": ["00C8"], + "FaultDriveMotorNum": 0, + "FaultDriveMotorList": [], + "FaultEngineNum": 1, + "FaultEngineList": ["006F"], + "FaultOthersNum": 0, + "FaultOthersList": [], + "GeneralAlarmFlag": 3, + "MaxAlarmLevel": 1, + "Type": "Alarm" + } + ], + "Time": { + "Day": 20, + "Hour": 22, + "Minute": 23, + "Month": 12, + "Second": 59, + "Year": 17 + } + } +} +``` + +其中，报警数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| --------------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Alarm` | +| `MaxAlarmLevel` | Integer | 最高报警等级，有效值范围 0~3，`0` 表示无故障，`1` 表示 `1` 级故障 | +| `GeneralAlarmFlag` | Integer | 通用报警标志位，见原协议表 18 | +| `FaultChargeableDeviceNum` | Integer | 可充电储能装置故障总数，有效值 0~252 | +| `FaultChargeableDeviceList` | Array | 可充电储能装置故障代码列表 | +| `FaultDriveMotorNum` | Integer | 驱动电机故障总数，有效置范围 0 ~252 | +| `FaultDriveMotorList` | Array | 驱动电机故障代码列表 | +| `FaultEngineNum` | Integer | 发动机故障总数，有效值范围 0~252 | +| `FaultEngineList` | Array | 发动机故障代码列表 | +| `FaultOthersNum` | Integer | 其他故障总数 | +| `FaultOthersList` | Array | 其他故障代码列表 | + + + +### 可充电储能装置电压数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "CellsTotal": 2, + "CellsVoltage": [5000], + "ChargeableCurrent": 10000, + "ChargeableSubsysNo": 1, + "ChargeableVoltage": 5000, + "FrameCellsCount": 1, + "FrameCellsIndex": 0 + }, + { + "CellsTotal": 2, + "CellsVoltage": [5001], + "ChargeableCurrent": 10001, + "ChargeableSubsysNo": 2, + "ChargeableVoltage": 5001, + "FrameCellsCount": 1, + "FrameCellsIndex": 1 + } + ], + "Type": "ChargeableVoltage" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +其中，字段定义如下 + +| 字段 | 类型 | 描述 | +| ----------- | ------- | ------------------------------------ | +| `Type` | String | 数据类型，此处位 `ChargeableVoltage` | +| `Number` | Integer | 可充电储能子系统个数，有效范围 1~250 | +| `SubSystem` | Object | 可充电储能子系统电压信息列表 | + +可充电储能子系统电压信息数据格式： + +| 字段 | 类型 | 描述 | +| -------------------- | ------- | ------------------------------------------------------------ | +| `ChargeableSubsysNo` | Integer | 可充电储能子系统号，有效值范围，1~250 | +| `ChargeableVoltage` | Integer | 可充电储能装置电压，有效值范围，0~10000（表示 0V~1000V）单位 0.1 V | +| `ChargeableCurrent` | Integer | 可充电储能装置电流，有效值范围，0~20000（数值偏移量 1000A，表示 -1000A~+1000A）单位 0.1 A | +| `CellsTotal` | Integer | 单体电池总数，有效值范围 1~65531 | +| `FrameCellsIndex` | Integer | 本帧起始电池序号，当本帧单体个数超过 200 时，应该拆分多个帧进行传输，有效值范围 1~65531 | +| `FrameCellsCount` | Integer | 本帧单体电池总数，有效值范围 1~200 | +| `CellsVoltage` | Array | 单体电池电压，有效值范围 0~60000（表示 0V~60.000V）单位 0.001V | + + + +### 可充电储能装置温度数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "ChargeableSubsysNo": 1, + "ProbeNum": 10, + "ProbesTemp": [0, 0, 0, 0, 0, 0, 0, 0, 19, 136] + }, + { + "ChargeableSubsysNo": 2, + "ProbeNum": 1, + "ProbesTemp": [100] + } + ], + "Type": "ChargeableTemp" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` +其中，数据格式为： + +| 字段 | 类型 | 描述 | +| ------------ | ------- | --------------------------------- | +| `Type` | String | 数据类型，此处为 `ChargeableTemp` | +| `Number` | Integer | 可充电储能子系统温度信息列表长度 | +| `SubSystems` | Object | 可充电储能子系统温度信息列表 | + +可充电储能子系统温度信息格式为 + +| 字段 | 类型 | 描述 | +| -------------------- | -------- | ------------------------------------ | +| `ChargeableSubsysNo` | Ineteger | 可充电储能子系统号，有效值 1~250 | +| `ProbeNum` | Integer | 可充电储能温度探针个数 | +| `ProbesTemp` | Array | 可充电储能子系统各温度探针温度值列表 | + + + +## 数据补发 + +Topic: gbt32960/${vin}/upstream/reinfo + +**数据格式: 略** (与实时数据上报相同) + +# Downstream + +> 请求数据流向: EMQX -> emqx_gbt32960 -> Terminal + +> 应答数据流向: Terminal -> emqx_gbt32960 -> EMQX + +下行主题: gbt32960/${vin}/dnstream +上行应答主题: gbt32960/${vin}/upstream/response + +## 参数查询 + + + +**Req:** + +```json +{ + "Action": "Query", + "Total": 2, + "Ids": ["0x01", "0x02"] +} +``` + +| 字段 | 类型 | 描述 | +| -------- | ------- | -------------------------------------------------- | +| `Action` | String | 下发命令类型，此处为 `Query` | +| `Total` | Integer | 查询参数总数 | +| `Ids` | Array | 需查询参数的 ID 列表，具体 ID 含义见原协议 表 B.10 | + +**Response:** + +```json +{ + "Cmd": 128, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 6000}, + {"0x02": 10} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + + + +## 参数设置 + +**Req:** +```json +{ + "Action": "Setting", + "Total": 2, + "Params": [{"0x01": 5000}, + {"0x02": 200}] +} +``` + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------ | +| `Action` | String | 下发命令类型，此处为 `Setting` | +| `Total` | Integer | 设置参数总数 | +| `Params` | Array | 需设置参数的 ID 和 值 | + +**Response:** + +```json +// fixme? 终端是按照这种方式返回? +{ + "Cmd": 129, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 5000}, + {"0x02": 200} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + +## 终端控制 +**命令的不同, 参数不同; 无参数时为空** + +远程升级: +**Req:** + +```json +{ + "Action": "Control", + "Command": "0x01", + "Param": { + "DialingName": "hz203", + "Username": "user001", + "Password": "password01", + "Ip": "192.168.199.1", + "Port": 8080, + "ManufacturerId": "BMWA", + "HardwareVer": "1.0.0", + "SoftwareVer": "1.0.0", + "UpgradeUrl": "ftp://emqtt.io/ftp/server", + "Timeout": 10 + } +} +``` + +| 字段 | 类型 | 描述 | +| --------- | ------- | ------------------------------ | +| `Action` | String | 下发命令类型，此处为 `Control` | +| `Command` | Integer | 下发指令 ID,见原协议表 B.15 | +| `Param` | Object | 命令参数 | + +列表 + +车载终端关机: + +```json +{ + "Action": "Control", + "Command": "0x02" +} +``` + +... + +车载终端报警: +```json +{ + "Action": "Control", + "Command": "0x06", + "Param": {"Level": 0, "Message": "alarm message"} +} +``` diff --git a/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl b/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl new file mode 100644 index 000000000..ce1a3f135 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-record(frame, {cmd, ack, vin, encrypt, length, data, check, rawdata}). + +-define(CMD(CmdType), #frame{ + cmd = CmdType, + ack = ?ACK_IS_CMD +}). + +-define(CMD(CmdType, Data), #frame{ + cmd = CmdType, + data = Data, + ack = ?ACK_IS_CMD +}). + +-define(IS_ACK_CODE(C), + (C == ?ACK_SUCCESS orelse + C == ?ACK_ERROR orelse + C == ?ACK_VIN_REPEAT) +). + +%%-------------------------------------------------------------------- +%% CMD Feilds +%%-------------------------------------------------------------------- +-define(CMD_VIHECLE_LOGIN, 16#01). +-define(CMD_INFO_REPORT, 16#02). +-define(CMD_INFO_RE_REPORT, 16#03). +-define(CMD_VIHECLE_LOGOUT, 16#04). +-define(CMD_PLATFORM_LOGIN, 16#05). +-define(CMD_PLATFORM_LOGOUT, 16#06). +-define(CMD_HEARTBEAT, 16#07). +-define(CMD_SCHOOL_TIME, 16#08). +% 0x09~0x7F: Reserved by upstream system +% 0x80~0x82: Reserved by terminal data +-define(CMD_PARAM_QUERY, 16#80). +-define(CMD_PARAM_SETTING, 16#81). +-define(CMD_TERMINAL_CTRL, 16#82). + +% 0x83~0xBF: Reserved by downstream system +% 0xC0~0xFE: Customized data for Platform Exchange Protocol + +%%-------------------------------------------------------------------- +%% ACK Feilds +%%-------------------------------------------------------------------- +-define(ACK_SUCCESS, 16#01). +-define(ACK_ERROR, 16#02). +-define(ACK_VIN_REPEAT, 16#03). +-define(ACK_IS_CMD, 16#FE). + +%%-------------------------------------------------------------------- +%% Encrypt Feilds +%%-------------------------------------------------------------------- +-define(ENCRYPT_NONE, 16#01). +-define(ENCRYPT_RSA, 16#02). +-define(ENCRYPT_AES128, 16#03). +-define(ENCRYPT_ABNORMAL, 16#FE). +-define(ENCRYPT_INVAILD, 16#FF). + +%%-------------------------------------------------------------------- +%% Info Type Flags +%%-------------------------------------------------------------------- +-define(INFO_TYPE_VEHICLE, 16#01). +-define(INFO_TYPE_DRIVE_MOTOR, 16#02). +-define(INFO_TYPE_FUEL_CELL, 16#03). +-define(INFO_TYPE_ENGINE, 16#04). +-define(INFO_TYPE_LOCATION, 16#05). +-define(INFO_TYPE_EXTREME, 16#06). +-define(INFO_TYPE_ALARM, 16#07). +-define(INFO_TYPE_CHARGEABLE_VOLTAGE, 16#08). +-define(INFO_TYPE_CHARGEABLE_TEMP, 16#09). +% 0x0A~0x2F: Customized data for Platform Exchange Protocol +% 0x30~0x7F: Reserved +% 0x80~0xFE: Customized by user diff --git a/apps/emqx_gateway_gbt32960/rebar.config b/apps/emqx_gateway_gbt32960/rebar.config new file mode 100644 index 000000000..cfeb0a195 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/rebar.config @@ -0,0 +1,6 @@ +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src new file mode 100644 index 000000000..ee6cf30d8 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src @@ -0,0 +1,10 @@ +{application, emqx_gateway_gbt32960, [ + {description, "GBT32960 Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["BSL"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl new file mode 100644 index 000000000..dde54026d --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl @@ -0,0 +1,97 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% @doc The GBT32960 Gateway implement +-module(emqx_gateway_gbt32960). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +%% define a gateway named gbt32960 +-gateway(#{ + name => gbt32960, + callback_module => ?MODULE, + config_schema_module => emqx_gbt32960_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + +-import( + emqx_gateway_utils, + [ + normalize_config/1, + start_listeners/4, + stop_listeners/2 + ] +). + +%%-------------------------------------------------------------------- +%% emqx_gateway_impl callbacks +%%-------------------------------------------------------------------- + +on_gateway_load( + _Gateway = #{ + name := GwName, + config := Config + }, + Ctx +) -> + Listeners = normalize_config(Config), + ModCfg = #{ + frame_mod => emqx_gbt32960_frame, + chann_mod => emqx_gbt32960_channel + }, + case + start_listeners( + Listeners, GwName, Ctx, ModCfg + ) + of + {ok, ListenerPids} -> + %% FIXME: How to throw an exception to interrupt the restart logic ? + %% FIXME: Assign ctx to GwState + {ok, ListenerPids, _GwState = #{ctx => Ctx}}; + {error, {Reason, Listener}} -> + throw( + {badconf, #{ + key => listeners, + value => Listener, + reason => Reason + }} + ) + end. + +on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> + GwName = maps:get(name, Gateway), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old state??? + on_gateway_unload(Gateway, GwState), + on_gateway_load(Gateway#{config => Config}, Ctx) + catch + Class:Reason:Stk -> + logger:error( + "Failed to update ~ts; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [GwName, Class, Reason, Stk] + ), + {error, Reason} + end. + +on_gateway_unload( + _Gateway = #{ + name := GwName, + config := Config + }, + _GwState +) -> + Listeners = normalize_config(Config), + stop_listeners(GwName, Listeners). diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl new file mode 100644 index 000000000..6b491a807 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl @@ -0,0 +1,862 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_channel). +-behaviour(emqx_gateway_channel). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). + +-export([ + info/1, + info/2, + stats/1 +]). + +-export([ + init/2, + handle_in/2, + handle_deliver/2, + handle_timeout/3, + terminate/2, + set_conn_state/2 +]). + +-export([ + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% ConnInfo + conninfo :: emqx_types:conninfo(), + %% ClientInfo + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: undefined | map(), + %% Keepalive + keepalive :: maybe(emqx_keepalive:keepalive()), + %% Conn State + conn_state :: conn_state(), + %% Timers + timers :: #{atom() => undefined | disabled | reference()}, + %% Inflight + inflight :: emqx_inflight:inflight(), + %% Message Queue + mqueue :: queue:queue(), + retx_interval, + retx_max_times, + max_mqueue_len +}). + +-type conn_state() :: idle | connecting | connected | disconnected. + +-type channel() :: #channel{}. + +-type reply() :: + {outgoing, emqx_types:packet()} + | {outgoing, [emqx_types:packet()]} + | {event, conn_state() | updated} + | {close, Reason :: atom()}. + +-type replies() :: reply() | [reply()]. +-type frame() :: emqx_gbt32960_frame:frame(). + +-define(TIMER_TABLE, #{ + alive_timer => keepalive, + retry_timer => retry_delivery +}). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). +-define(DEFAULT_MOUNTPOINT, <<"gbt32960/${clientid}">>). +-define(DEFAULT_DOWNLINK_TOPIC, <<"/dnstream">>). + +-dialyzer({nowarn_function, init/2}). + +%%-------------------------------------------------------------------- +%% Info, Attrs and Caps +%%-------------------------------------------------------------------- + +%% @doc Get infos of the channel. +-spec info(channel()) -> emqx_types:infos(). +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +-spec info(list(atom()) | atom(), channel()) -> term(). +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(zone, #channel{clientinfo = #{zone := Zone}}) -> + Zone; +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, _) -> + #{}; +info(conn_state, #channel{conn_state = ConnState}) -> + ConnState; +info(keepalive, #channel{keepalive = undefined}) -> + undefined; +info(keepalive, #channel{keepalive = Keepalive}) -> + emqx_keepalive:info(Keepalive); +info(will_msg, _) -> + undefined. + +-spec stats(channel()) -> emqx_types:stats(). +stats(#channel{inflight = Inflight, mqueue = Queue}) -> + %% XXX: A fake stats for managed by emqx_management + [ + {subscriptions_cnt, 1}, + {subscriptions_max, 0}, + {inflight_cnt, emqx_inflight:size(Inflight)}, + {inflight_max, emqx_inflight:max_size(Inflight)}, + {mqueue_len, queue:len(Queue)}, + {mqueue_max, 0}, + {mqueue_dropped, 0}, + {next_pkt_id, 0}, + {awaiting_rel_cnt, 0}, + {awaiting_rel_max, 0} + ]. + +set_conn_state(ConnState, Channel) -> + Channel#channel{conn_state = ConnState}. + +%%-------------------------------------------------------------------- +%% Init the Channel +%%-------------------------------------------------------------------- + +init( + ConnInfo = #{ + peername := {PeerHost, _Port}, + sockname := {_Host, SockPort} + }, + Options +) -> + % TODO: init rsa_key from user input + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Options, ?DEFAULT_MOUNTPOINT), + ListenerId = + case maps:get(listener, Options, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, + EnableAuthn = maps:get(enable_authn, Options, true), + + ClientInfo = setting_peercert_infos( + Peercert, + #{ + zone => default, + listener => ListenerId, + protocol => gbt32960, + peerhost => PeerHost, + sockport => SockPort, + clientid => undefined, + username => undefined, + is_bridge => false, + is_superuser => false, + enable_authn => EnableAuthn, + mountpoint => Mountpoint + } + ), + + Ctx = maps:get(ctx, Options), + + #{ + retry_interval := RetxInterv, + max_retry_times := RetxMaxTime, + message_queue_len := MessageQueueLen + } = Options, + + #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo, + inflight = emqx_inflight:new(1), + mqueue = queue:new(), + timers = #{}, + conn_state = idle, + retx_interval = RetxInterv, + retx_max_times = RetxMaxTime, + max_mqueue_len = MessageQueueLen + }. + +setting_peercert_infos(NoSSL, ClientInfo) when + NoSSL =:= nossl; + NoSSL =:= undefined +-> + ClientInfo; +setting_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- +-spec handle_in(emqx_gbt32960_frame:frame() | {frame_error, any()}, channel()) -> + {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +handle_in( + Frame = ?CMD(?CMD_VIHECLE_LOGIN), + Channel +) -> + case + emqx_utils:pipeline( + [ + fun enrich_clientinfo/2, + fun enrich_conninfo/2, + fun set_log_meta/2, + %% TODO: How to implement the banned in the gateway instance? + %, fun check_banned/2 + fun auth_connect/2 + ], + Frame, + Channel#channel{conn_state = connecting} + ) + of + {ok, _NPacket, NChannel} -> + process_connect(Frame, ensure_connected(NChannel)); + {error, ReasonCode, NChannel} -> + log(warning, #{msg => "login_failed", reason => ReasonCode}, NChannel), + shutdown(ReasonCode, NChannel) + end; +handle_in(_Frame, Channel = #channel{conn_state = ConnState}) when + ConnState =/= connected +-> + shutdown(protocol_error, Channel); +handle_in(Frame = ?CMD(?CMD_INFO_REPORT), Channel) -> + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_INFO_RE_REPORT), Channel) -> + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_VIHECLE_LOGOUT), Channel) -> + %% XXX: unsubscribe gbt32960/dnstream/${vin}? + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_PLATFORM_LOGIN), Channel) -> + #{ + <<"Username">> := _Username, + <<"Password">> := _Password + } = Frame#frame.data, + %% TODO: + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_PLATFORM_LOGOUT), Channel) -> + %% TODO: + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_HEARTBEAT), Channel) -> + handle_out({?ACK_SUCCESS, Frame}, Channel); +handle_in(Frame = ?CMD(?CMD_SCHOOL_TIME), Channel) -> + %% TODO: How verify this request + handle_out({?ACK_SUCCESS, Frame}, Channel); +handle_in(Frame = #frame{cmd = Cmd}, Channel = #channel{inflight = Inflight}) -> + {Outgoings, NChannel} = dispatch_frame(Channel#channel{inflight = ack_frame(Cmd, Inflight)}), + _ = upstreaming(Frame, NChannel), + {ok, [{outgoing, Outgoings}], NChannel}; +handle_in(Frame, Channel) -> + log(warning, #{msg => "unexcepted_frame", frame => Frame}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle out +%%-------------------------------------------------------------------- + +handle_out({AckCode, Frame}, Channel) when + ?IS_ACK_CODE(AckCode) +-> + {ok, [{outgoing, ack(AckCode, Frame)}], Channel}. + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- +-spec handle_deliver(list(emqx_types:deliver()), channel()) -> + {ok, channel()} + | {ok, replies(), channel()}. + +handle_deliver( + Messages0, + Channel = #channel{ + clientinfo = #{clientid := ClientId, mountpoint := Mountpoint}, + mqueue = Queue, + max_mqueue_len = MaxQueueLen + } +) -> + Messages = lists:map( + fun({deliver, _, M}) -> + emqx_mountpoint:unmount(Mountpoint, M) + end, + Messages0 + ), + case MaxQueueLen - queue:len(Queue) of + N when N =< 0 -> + discard_downlink_messages(Messages, Channel), + {ok, Channel}; + N -> + {NMessages, Dropped} = split_by_pos(Messages, N), + log(debug, #{msg => "enqueue_messages", messages => NMessages}, Channel), + metrics_inc('messages.delivered', Channel, erlang:length(NMessages)), + discard_downlink_messages(Dropped, Channel), + Frames = msgs2frame(NMessages, ClientId, Channel), + NQueue = lists:foldl(fun(F, Q) -> queue:in(F, Q) end, Queue, Frames), + {Outgoings, NChannel} = dispatch_frame(Channel#channel{mqueue = NQueue}), + {ok, [{outgoing, Outgoings}], NChannel} + end. + +split_by_pos(L, Pos) -> + split_by_pos(L, Pos, []). + +split_by_pos([], _, A1) -> + {lists:reverse(A1), []}; +split_by_pos(L, 0, A1) -> + {lists:reverse(A1), L}; +split_by_pos([E | L], N, A1) -> + split_by_pos(L, N - 1, [E | A1]). + +msgs2frame(Messages, Vin, Channel) -> + lists:filtermap( + fun(#message{payload = Payload}) -> + case emqx_utils_json:safe_decode(Payload, [return_maps]) of + {ok, Maps} -> + case msg2frame(Maps, Vin) of + {error, Reason} -> + log( + debug, + #{ + msg => "convert_message_to_frame_error", + reason => Reason, + data => Maps + }, + Channel + ), + false; + Frame -> + {true, Frame} + end; + {error, Reason} -> + log(error, #{msg => "json_decode_error", reason => Reason}, Channel), + false + end + end, + Messages + ). + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- + +-spec handle_call(Req :: term(), From :: term(), channel()) -> + {reply, Reply :: term(), channel()} + | {reply, Reply :: term(), replies(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), frame(), channel()}. + +handle_call(kick, _From, Channel) -> + Channel1 = ensure_disconnected(kicked, Channel), + disconnect_and_shutdown(kicked, ok, Channel1); +handle_call(discard, _From, Channel) -> + disconnect_and_shutdown(discarded, ok, Channel); +handle_call(Req, _From, Channel) -> + log(error, #{msg => "unexpected_call", call => Req}, Channel), + reply(ignored, Channel). + +%%-------------------------------------------------------------------- +%% Handle cast +%%-------------------------------------------------------------------- + +-spec handle_cast(Req :: term(), channel()) -> + ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. +handle_cast(_Req, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle info +%%-------------------------------------------------------------------- + +-spec handle_info(Info :: term(), channel()) -> + ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. + +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = idle}) -> + shutdown(Reason, Channel); +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = connecting}) -> + shutdown(Reason, Channel); +handle_info( + {sock_closed, Reason}, + Channel = + #channel{ + conn_state = connected + } +) -> + NChannel = ensure_disconnected(Reason, Channel), + shutdown(Reason, NChannel); +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = disconnected}) -> + log(error, #{msg => "unexpected_sock_closed", reason => Reason}, Channel), + {ok, Channel}; +handle_info(Info, Channel) -> + log(error, #{msg => "unexpected_info}", info => Info}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- + +-spec handle_timeout(reference(), Msg :: term(), channel()) -> + {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}. + +handle_timeout( + _TRef, + {keepalive, _StatVal}, + Channel = #channel{keepalive = undefined} +) -> + {ok, Channel}; +handle_timeout( + _TRef, + {keepalive, _StatVal}, + Channel = #channel{conn_state = disconnected} +) -> + {ok, Channel}; +handle_timeout( + _TRef, + {keepalive, StatVal}, + Channel = #channel{keepalive = Keepalive} +) -> + case emqx_keepalive:check(StatVal, Keepalive) of + {ok, NKeepalive} -> + NChannel = Channel#channel{keepalive = NKeepalive}, + {ok, reset_timer(alive_timer, NChannel)}; + {error, timeout} -> + shutdown(keepalive_timeout, Channel) + end; +handle_timeout( + _TRef, + retry_delivery, + Channel = #channel{inflight = Inflight, retx_interval = RetxInterv} +) -> + case emqx_inflight:is_empty(Inflight) of + true -> + {ok, clean_timer(retry_timer, Channel)}; + false -> + Frames = emqx_inflight:to_list(Inflight), + {Outgoings, NInflight} = retry_delivery( + Frames, erlang:system_time(millisecond), RetxInterv, Inflight, [] + ), + {Outgoings2, NChannel} = dispatch_frame(Channel#channel{inflight = NInflight}), + {ok, [{outgoing, Outgoings ++ Outgoings2}], reset_timer(retry_timer, NChannel)} + end; +handle_timeout(_TRef, Msg, Channel) -> + log(error, #{msg => "unexpected_timeout", content => Msg}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Ensure timers +%%-------------------------------------------------------------------- + +ensure_timer(Name, Channel = #channel{timers = Timers}) -> + TRef = maps:get(Name, Timers, undefined), + Time = interval(Name, Channel), + case TRef == undefined andalso Time > 0 of + true -> ensure_timer(Name, Time, Channel); + %% Timer disabled or exists + false -> Channel + end. + +ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> + log(debug, #{msg => "start_timer", name => Name, time => Time}, Channel), + Msg = maps:get(Name, ?TIMER_TABLE), + TRef = emqx_utils:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +reset_timer(Name, Channel) -> + ensure_timer(Name, clean_timer(Name, Channel)). + +clean_timer(Name, Channel = #channel{timers = Timers}) -> + Channel#channel{timers = maps:remove(Name, Timers)}. + +interval(alive_timer, #channel{keepalive = KeepAlive}) -> + emqx_keepalive:info(interval, KeepAlive); +interval(retry_timer, #channel{retx_interval = RetxIntv}) -> + RetxIntv. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- + +terminate(Reason, #channel{ + ctx = Ctx, + session = Session, + clientinfo = ClientInfo +}) -> + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). + +%%-------------------------------------------------------------------- +%% Ensure connected + +enrich_clientinfo( + Packet, + Channel = #channel{ + clientinfo = ClientInfo + } +) -> + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( + [ + fun maybe_assign_clientid/2, + %% FIXME: CALL After authentication successfully + fun fix_mountpoint/2 + ], + Packet, + ClientInfo + ), + {ok, NPacket, Channel#channel{clientinfo = NClientInfo}}. + +enrich_conninfo( + _Packet, + Channel = #channel{ + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + #{clientid := ClientId, username := Username} = ClientInfo, + NConnInfo = ConnInfo#{ + proto_name => <<"GBT32960">>, + proto_ver => <<"">>, + clean_start => true, + keepalive => 0, + expiry_interval => 0, + conn_props => #{}, + receive_maximum => 0, + clientid => ClientId, + username => Username + }, + {ok, Channel#channel{conninfo = NConnInfo}}. + +set_log_meta(_Packet, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect( + _Packet, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + } +) -> + #{ + clientid := ClientId, + username := Username + } = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "client_login_failed", + clientid => ClientId, + username => Username, + reason => Reason + }), + {error, Reason} + end. + +ensure_connected( + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{ + conninfo = NConnInfo, + conn_state = connected + }. + +process_connect( + Frame, + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + SessFun = fun(_, _) -> #{} end, + case + emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) + of + {ok, #{session := Session}} -> + NChannel = Channel#channel{session = Session}, + subscribe_downlink(?DEFAULT_DOWNLINK_TOPIC, Channel), + _ = upstreaming(Frame, NChannel), + %% XXX: connection_accepted is not defined by stomp protocol + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, #{}]), + {ok, NChannel}; + {error, Reason} -> + log( + error, + #{ + msg => "failed_to_open_session", + reason => Reason + }, + Channel + ), + shutdown(Reason, Channel) + end. + +maybe_assign_clientid(#frame{vin = Vin}, ClientInfo) -> + {ok, ClientInfo#{clientid => Vin, username => Vin}}. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> + ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the variable replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +%%-------------------------------------------------------------------- +%% Ensure disconnected + +ensure_disconnected( + Reason, + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks( + Ctx, + 'client.disconnected', + [ClientInfo, Reason, NConnInfo] + ), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +reply(Reply, Channel) -> + {reply, Reply, Channel}. + +shutdown(Reason, Channel) -> + {shutdown, Reason, Channel}. + +shutdown(Reason, Reply, Channel) -> + {shutdown, Reason, Reply, Channel}. + +disconnect_and_shutdown(Reason, Reply, Channel) -> + shutdown(Reason, Reply, Channel). + +retry_delivery([], _Now, _Interval, Inflight, Acc) -> + {lists:reverse(Acc), Inflight}; +retry_delivery([{Key, {_Frame, 0, _}} | Frames], Now, Interval, Inflight, Acc) -> + %% todo log(error, "has arrived max re-send times, drop ~p", [Frame]), + NInflight = emqx_inflight:delete(Key, Inflight), + retry_delivery(Frames, Now, Interval, NInflight, Acc); +retry_delivery([{Key, {Frame, RetxCount, Ts}} | Frames], Now, Interval, Inflight, Acc) -> + Diff = Now - Ts, + case Diff >= Interval of + true -> + NInflight = emqx_inflight:update(Key, {Frame, RetxCount - 1, Now}, Inflight), + retry_delivery(Frames, Now, Interval, NInflight, [Frame | Acc]); + _ -> + retry_delivery(Frames, Now, Interval, Inflight, Acc) + end. + +upstreaming( + Frame, Channel = #channel{clientinfo = #{mountpoint := Mountpoint, clientid := ClientId}} +) -> + {Topic, Payload} = transform(Frame, Mountpoint), + log(debug, #{msg => "upstreaming_to_topic", topic => Topic, payload => Payload}, Channel), + emqx:publish(emqx_message:make(ClientId, ?QOS_1, Topic, Payload)). + +transform(Frame = ?CMD(Cmd), Mountpoint) -> + Suffix = + case Cmd of + ?CMD_VIHECLE_LOGIN -> <<"/upstream/vlogin">>; + ?CMD_INFO_REPORT -> <<"/upstream/info">>; + ?CMD_INFO_RE_REPORT -> <<"/upstream/reinfo">>; + ?CMD_VIHECLE_LOGOUT -> <<"/upstream/vlogout">>; + ?CMD_PLATFORM_LOGIN -> <<"/upstream/plogin">>; + ?CMD_PLATFORM_LOGOUT -> <<"/upstream/plogout">>; + %CMD_HEARTBEAT, CMD_SCHOOL_TIME ... + _ -> <<"/upstream/transparent">> + end, + Topic = emqx_mountpoint:mount(Mountpoint, Suffix), + Payload = to_json(Frame), + {Topic, Payload}; +transform(Frame = #frame{ack = Ack}, Mountpoint) when + ?IS_ACK_CODE(Ack) +-> + Topic = emqx_mountpoint:mount(Mountpoint, <<"/upstream/response">>), + Payload = to_json(Frame), + {Topic, Payload}. + +to_json(#frame{cmd = Cmd, vin = Vin, encrypt = Encrypt, data = Data}) -> + emqx_utils_json:encode(#{'Cmd' => Cmd, 'Vin' => Vin, 'Encrypt' => Encrypt, 'Data' => Data}). + +ack(Code, Frame = #frame{data = Data, ack = ?ACK_IS_CMD}) -> + % PROTO: Update time & ack feilds only + Frame#frame{ack = Code, data = Data#{<<"Time">> => gentime()}}. + +ack_frame(Key, Inflight) -> + case emqx_inflight:contain(Key, Inflight) of + true -> emqx_inflight:delete(Key, Inflight); + false -> Inflight + end. + +dispatch_frame( + Channel = #channel{ + mqueue = Queue, + inflight = Inflight, + retx_max_times = RetxMax + } +) -> + case emqx_inflight:is_full(Inflight) orelse queue:is_empty(Queue) of + true -> + {[], Channel}; + false -> + {{value, Frame}, NewQueue} = queue:out(Queue), + + log(debug, #{msg => "delivery", frame => Frame}, Channel), + + NewInflight = emqx_inflight:insert( + Frame#frame.cmd, {Frame, RetxMax, erlang:system_time(millisecond)}, Inflight + ), + NChannel = Channel#channel{mqueue = NewQueue, inflight = NewInflight}, + {[Frame], ensure_timer(retry_timer, NChannel)} + end. + +gentime() -> + {Year, Mon, Day} = date(), + {Hour, Min, Sec} = time(), + Year1 = list_to_integer(string:substr(integer_to_list(Year), 3, 2)), + #{ + <<"Year">> => Year1, + <<"Month">> => Mon, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Min, + <<"Second">> => Sec + }. + +%%-------------------------------------------------------------------- +%% Message to frame +%%-------------------------------------------------------------------- + +msg2frame(#{<<"Action">> := <<"Query">>, <<"Total">> := Total, <<"Ids">> := Ids}, Vin) -> + % Ids = [<<"0x01">>, <<"0x02">>] --> [1, 2] + Data = #{ + <<"Time">> => gentime(), + <<"Total">> => Total, + <<"Ids">> => lists:map(fun hexstring_to_byte/1, Ids) + }, + #frame{ + cmd = ?CMD_PARAM_QUERY, ack = ?ACK_IS_CMD, vin = Vin, encrypt = ?ENCRYPT_NONE, data = Data + }; +msg2frame(#{<<"Action">> := <<"Setting">>, <<"Total">> := Total, <<"Params">> := Params}, Vin) -> + % Params = [#{<<"0x01">> := 5000}, #{<<"0x02">> := 400}] + % Params1 = [#{1 := 5000}, #{2 := 400}] + Params1 = lists:foldr( + fun(M, Acc) -> + [{K, V}] = maps:to_list(M), + [#{hexstring_to_byte(K) => V} | Acc] + end, + [], + Params + ), + Data = #{<<"Time">> => gentime(), <<"Total">> => Total, <<"Params">> => Params1}, + #frame{ + cmd = ?CMD_PARAM_SETTING, ack = ?ACK_IS_CMD, vin = Vin, encrypt = ?ENCRYPT_NONE, data = Data + }; +msg2frame(Data = #{<<"Action">> := <<"Control">>, <<"Command">> := Command}, Vin) -> + Param = maps:get(<<"Param">>, Data, <<>>), + Data1 = #{ + <<"Time">> => gentime(), + <<"Command">> => hexstring_to_byte(Command), + <<"Param">> => Param + }, + #frame{ + cmd = ?CMD_TERMINAL_CTRL, + ack = ?ACK_IS_CMD, + vin = Vin, + encrypt = ?ENCRYPT_NONE, + data = Data1 + }; +msg2frame(_Data, _Vin) -> + {error, unsupproted}. + +hexstring_to_byte(S) when is_binary(S) -> + hexstring_to_byte(binary_to_list(S)); +hexstring_to_byte("0x" ++ S) -> + tune_byte(list_to_integer(S, 16)); +hexstring_to_byte(S) -> + tune_byte(list_to_integer(S)). + +tune_byte(I) when I =< 16#FF -> I; +tune_byte(_) -> exit(invalid_byte). + +discard_downlink_messages([], _Channel) -> + ok; +discard_downlink_messages(Messages, Channel) -> + log( + error, + #{ + msg => "discard_new_downlink_messages", + reason => + "Discard new downlink messages due to that too" + " many messages are waiting their ACKs.", + messages => Messages + }, + Channel + ), + metrics_inc('delivery.dropped', Channel, erlang:length(Messages)). + +log(Level, Meta, #channel{clientinfo = #{clientid := ClientId, username := Username}} = _Channel) -> + ?SLOG(Level, Meta#{clientid => ClientId, username => Username}). + +metrics_inc(Name, #channel{ctx = Ctx}, Oct) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name, Oct). + +subscribe_downlink( + Topic, + #channel{ + ctx = Ctx, + clientinfo = + ClientInfo = + #{ + clientid := ClientId, + mountpoint := Mountpoint + } + } +) -> + {ParsedTopic, SubOpts0} = emqx_topic:parse(Topic), + SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), SubOpts0), + MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), + _ = emqx_broker:subscribe(MountedTopic, ClientId, SubOpts), + run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]). diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl new file mode 100644 index 000000000..a5ec3fa77 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl @@ -0,0 +1,802 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_frame). + +-behaviour(emqx_gateway_frame). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% emqx_gateway_frame callbacks +-export([ + initial_parse_state/1, + serialize_opts/0, + serialize_pkt/2, + parse/2, + format/1, + type/1, + is_message/1 +]). + +-define(FLAG, 1 / binary). +-define(BYTE, 8 / big - integer). +-define(WORD, 16 / big - integer). +-define(DWORD, 32 / big - integer). +%% CMD: 1, ACK: 1, VIN: 17, Enc: 1, Len: 2 +-define(HEADER_SIZE, 22). + +-define(IS_RESPONSE(Ack), + Ack == ?ACK_SUCCESS orelse + Ack == ?ACK_ERROR orelse + Ack == ?ACK_VIN_REPEAT +). + +-type phase() :: search_heading_0x23 | parse. + +-type parser_state() :: #{ + data := binary(), + phase := phase() +}. + +%%-------------------------------------------------------------------- +%% Init a Parser +%%-------------------------------------------------------------------- + +-spec initial_parse_state(map()) -> parser_state(). +initial_parse_state(_) -> + #{data => <<>>, phase => search_heading_0x23}. + +-spec serialize_opts() -> emqx_gateway_frame:serialize_options(). +serialize_opts() -> + #{}. + +%%-------------------------------------------------------------------- +%% Parse Message +%%-------------------------------------------------------------------- +parse(Bin, State) -> + case enter_parse(Bin, State) of + {ok, Message, Rest} -> + {ok, Message, Rest, State#{parse => search_heading_0x23}}; + {error, Error} -> + {error, Error}; + {more_data_follow, Partial} -> + {more, State#{data => Partial, phase => parse}} + end. + +enter_parse(Bin, #{phase := search_heading_0x23}) -> + case search_heading_0x23(Bin) of + {ok, Rest} -> + parse_msg(Rest); + Error -> + Error + end; +enter_parse(Bin, #{data := Data}) -> + parse_msg(<>). + +search_heading_0x23(<<16#23, 16#23, Rest/binary>>) -> + {ok, Rest}; +search_heading_0x23(<<_, Rest/binary>>) -> + search_heading_0x23(Rest); +search_heading_0x23(<<>>) -> + {error, invalid_frame}. + +parse_msg(Binary) -> + case byte_size(Binary) >= ?HEADER_SIZE of + true -> + {Frame, Rest2} = parse_header(Binary), + case byte_size(Rest2) >= Frame#frame.length + 1 of + true -> parse_body(Rest2, Frame); + false -> {more_data_follow, Binary} + end; + false -> + {more_data_follow, Binary} + end. + +parse_header(<> = Binary) -> + Check = cal_check(Binary, ?HEADER_SIZE, undefined), + { + #frame{cmd = Cmd, ack = Ack, vin = VIN, encrypt = Encrypt, length = Length, check = Check}, + Rest2 + }. + +parse_body(Binary, Frame = #frame{length = Length, check = OldCheck, encrypt = Encrypt}) -> + <> = Binary, + Check = cal_check(Binary, Length, OldCheck), + case CheckByte == Check of + true -> + RawData = decipher(Data, Encrypt), + {ok, Frame#frame{data = parse_data(Frame, RawData), rawdata = RawData}, Rest}; + false -> + {error, frame_check_error} + end. + +% Algo: ?ENCRYPT_NONE, ENCRYPT_RSA, ENCRYPT_AES128 +decipher(Data, _Algo) -> + % TODO: decypher data + Data. + +% Algo: ?ENCRYPT_NONE, ENCRYPT_RSA, ENCRYPT_AES128 +encipher(Data, _Algo) -> + % TODO: encipher data + Data. + +parse_data( + #frame{cmd = ?CMD_VIHECLE_LOGIN}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq, + <<"ICCID">> => ICCID, + <<"Num">> => Num, + <<"Length">> => Length, + <<"Id">> => Id + }; +parse_data( + #frame{cmd = ?CMD_INFO_REPORT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Infos">> => parse_info(Infos, []) + }; +parse_data( + #frame{cmd = ?CMD_INFO_RE_REPORT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Infos">> => parse_info(Infos, []) + }; +parse_data( + #frame{cmd = ?CMD_VIHECLE_LOGOUT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq + }; +parse_data( + #frame{cmd = ?CMD_PLATFORM_LOGIN}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq, + <<"Username">> => Username, + <<"Password">> => Password, + <<"Encrypt">> => Encrypt + }; +parse_data( + #frame{cmd = ?CMD_PLATFORM_LOGOUT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq + }; +parse_data(#frame{cmd = ?CMD_HEARTBEAT}, <<>>) -> + #{}; +parse_data(#frame{cmd = ?CMD_SCHOOL_TIME}, <<>>) -> + #{}; +parse_data( + #frame{cmd = ?CMD_PARAM_QUERY}, + <> +) -> + %% XXX: need check ACK filed? + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Total">> => Total, + <<"Params">> => parse_params(Rest) + }; +parse_data( + #frame{cmd = ?CMD_PARAM_SETTING}, + <> +) -> + ?SLOG(debug, #{msg => "rest", data => Rest}), + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Total">> => Total, + <<"Params">> => parse_params(Rest) + }; +parse_data( + #frame{cmd = ?CMD_TERMINAL_CTRL}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Command">> => Command, + <<"Param">> => parse_ctrl_param(Command, Rest) + }; +parse_data(Frame, Data) -> + ?SLOG(error, #{msg => "invalid_frame", frame => Frame, data => Data}), + error(invalid_frame). + +%%-------------------------------------------------------------------- +%% Parse Report Data Info +%%-------------------------------------------------------------------- + +parse_info(<<>>, Acc) -> + lists:reverse(Acc); +parse_info(<>, Acc) -> + <> = Body, + parse_info(Rest, [ + #{ + <<"Type">> => <<"Vehicle">>, + <<"Status">> => Status, + <<"Charging">> => Charging, + <<"Mode">> => Mode, + <<"Speed">> => Speed, + <<"Mileage">> => Mileage, + <<"Voltage">> => Voltage, + <<"Current">> => Current, + <<"SOC">> => SOC, + <<"DC">> => DC, + <<"Gear">> => Gear, + <<"Resistance">> => Resistance, + <<"AcceleratorPedal">> => AcceleratorPedal, + <<"BrakePedal">> => BrakePedal + } + | Acc + ]); +parse_info(<>, Acc) -> + % 12 is packet len of per drive motor + Len = Number * 12, + <> = Rest, + parse_info(Rest1, [ + #{ + <<"Type">> => <<"DriveMotor">>, + <<"Number">> => Number, + <<"Motors">> => parse_drive_motor(Bodys, []) + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = + Rest, + + <> = Rest1, + + <> = Rest2, + parse_info(Rest3, [ + #{ + <<"Type">> => <<"FuelCell">>, + <<"CellVoltage">> => CellVoltage, + <<"CellCurrent">> => CellCurrent, + <<"FuelConsumption">> => FuelConsumption, + <<"ProbeNum">> => ProbeNum, + <<"ProbeTemps">> => binary_to_list(ProbeTemps), + <<"H_MaxTemp">> => H_MaxTemp, + <<"H_TempProbeCode">> => H_TempProbeCode, + <<"H_MaxConc">> => H_MaxConc, + <<"H_ConcSensorCode">> => H_ConcSensorCode, + <<"H_MaxPress">> => H_MaxPress, + <<"H_PressSensorCode">> => H_PressSensorCode, + <<"DCStatus">> => DCStatus + } + | Acc + ]); +parse_info( + <>, + Acc +) -> + parse_info(Rest, [ + #{ + <<"Type">> => <<"Engine">>, + <<"Status">> => Status, + <<"CrankshaftSpeed">> => CrankshaftSpeed, + <<"FuelConsumption">> => FuelConsumption + } + | Acc + ]); +parse_info( + <>, Acc +) -> + parse_info(Rest, [ + #{ + <<"Type">> => <<"Location">>, + <<"Status">> => Status, + <<"Longitude">> => Longitude, + <<"Latitude">> => Latitude + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = Body, + + parse_info(Rest, [ + #{ + <<"Type">> => <<"Extreme">>, + <<"MaxVoltageBatterySubsysNo">> => MaxVoltageBatterySubsysNo, + <<"MaxVoltageBatteryCode">> => MaxVoltageBatteryCode, + <<"MaxBatteryVoltage">> => MaxBatteryVoltage, + <<"MinVoltageBatterySubsysNo">> => MinVoltageBatterySubsysNo, + <<"MinVoltageBatteryCode">> => MinVoltageBatteryCode, + <<"MinBatteryVoltage">> => MinBatteryVoltage, + <<"MaxTempSubsysNo">> => MaxTempSubsysNo, + <<"MaxTempProbeNo">> => MaxTempProbeNo, + <<"MaxTemp">> => MaxTemp, + <<"MinTempSubsysNo">> => MinTempSubsysNo, + <<"MinTempProbeNo">> => MinTempProbeNo, + <<"MinTemp">> => MinTemp + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = + Rest, + N1 = FaultChargeableDeviceNum * 4, + <> = Rest1, + N2 = FaultDriveMotorNum * 4, + <> = Rest2, + N3 = FaultEngineNum * 4, + <> = Rest3, + N4 = FaultOthersNum * 4, + <> = Rest4, + parse_info(Rest5, [ + #{ + <<"Type">> => <<"Alarm">>, + <<"MaxAlarmLevel">> => MaxAlarmLevel, + <<"GeneralAlarmFlag">> => GeneralAlarmFlag, + <<"FaultChargeableDeviceNum">> => FaultChargeableDeviceNum, + <<"FaultChargeableDeviceList">> => tune_fault_codelist(FaultChargeableDeviceList), + <<"FaultDriveMotorNum">> => FaultDriveMotorNum, + <<"FaultDriveMotorList">> => tune_fault_codelist(FaultDriveMotorList), + <<"FaultEngineNum">> => FaultEngineNum, + <<"FaultEngineList">> => tune_fault_codelist(FaultEngineList), + <<"FaultOthersNum">> => FaultOthersNum, + <<"FaultOthersList">> => tune_fault_codelist(FaultOthersList) + } + | Acc + ]); +parse_info(<>, Acc) -> + {Rest1, SubSystems} = parse_chargeable_voltage(Rest, Number, []), + parse_info(Rest1, [ + #{ + <<"Type">> => <<"ChargeableVoltage">>, + <<"Number">> => Number, + <<"SubSystems">> => SubSystems + } + | Acc + ]); +parse_info(<>, Acc) -> + {Rest1, SubSystems} = parse_chargeable_temp(Rest, Number, []), + parse_info(Rest1, [ + #{ + <<"Type">> => <<"ChargeableTemp">>, + <<"Number">> => Number, + <<"SubSystems">> => SubSystems + } + | Acc + ]); +parse_info(Rest, Acc) -> + ?SLOG(error, #{msg => "invalid_info_feild", rest => Rest, acc => Acc}), + error(invalid_info_feild). + +parse_drive_motor(<<>>, Acc) -> + lists:reverse(Acc); +parse_drive_motor( + <>, + Acc +) -> + parse_drive_motor(Rest, [ + #{ + <<"No">> => No, + <<"Status">> => Status, + <<"CtrlTemp">> => CtrlTemp, + <<"Rotating">> => Rotating, + <<"Torque">> => Torque, + <<"MotorTemp">> => MotorTemp, + <<"InputVoltage">> => InputVoltage, + <<"DCBusCurrent">> => DCBusCurrent + } + | Acc + ]). + +parse_chargeable_voltage(Rest, 0, Acc) -> + {Rest, lists:reverse(Acc)}; +parse_chargeable_voltage( + <>, + Num, + Acc +) -> + Len = FrameCellsCount * 2, + <> = Rest, + parse_chargeable_voltage(Rest1, Num - 1, [ + #{ + <<"ChargeableSubsysNo">> => ChargeableSubsysNo, + <<"ChargeableVoltage">> => ChargeableVoltage, + <<"ChargeableCurrent">> => ChargeableCurrent, + <<"CellsTotal">> => CellsTotal, + <<"FrameCellsIndex">> => FrameCellsIndex, + <<"FrameCellsCount">> => FrameCellsCount, + <<"CellsVoltage">> => tune_voltage(CellsVoltage) + } + | Acc + ]). + +parse_chargeable_temp(Rest, 0, Acc) -> + {Rest, lists:reverse(Acc)}; +parse_chargeable_temp(<>, Num, Acc) -> + <> = Rest, + parse_chargeable_temp(Rest1, Num - 1, [ + #{ + <<"ChargeableSubsysNo">> => ChargeableSubsysNo, + <<"ProbeNum">> => ProbeNum, + <<"ProbesTemp">> => binary_to_list(ProbesTemp) + } + | Acc + ]). +tune_fault_codelist(<<>>) -> + []; +tune_fault_codelist(Data) -> + lists:flatten([list_to_binary(io_lib:format("~4.16.0B", [X])) || <> <= Data]). + +tune_voltage(Bin) -> tune_voltage_(Bin, []). +tune_voltage_(<<>>, Acc) -> lists:reverse(Acc); +tune_voltage_(<>, Acc) -> tune_voltage_(Rest, [V | Acc]). + +parse_params(Bin) -> parse_params_(Bin, []). +parse_params_(<<>>, Acc) -> + lists:reverse(Acc); +parse_params_(<<16#01, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x01">> => Val} | Acc]); +parse_params_(<<16#02, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x02">> => Val} | Acc]); +parse_params_(<<16#03, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x03">> => Val} | Acc]); +parse_params_(<<16#04, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x04">> => Val} | Acc]); +parse_params_(<<16#05, Rest/binary>>, Acc) -> + case [V || #{<<"0x04">> := V} <- Acc] of + [Len] -> + <> = Rest, + parse_params_(Rest1, [#{<<"0x05">> => Val} | Acc]); + _ -> + ?SLOG(error, #{ + msg => "invalid_data", reason => "cmd_0x04 must appear ahead of cmd_0x05" + }), + lists:reverse(Acc) + end; +parse_params_(<<16#06, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x06">> => Val} | Acc]); +parse_params_(<<16#07, Val:5/binary, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x07">> => Val} | Acc]); +parse_params_(<<16#08, Val:5/binary, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x08">> => Val} | Acc]); +parse_params_(<<16#09, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x09">> => Val} | Acc]); +parse_params_(<<16#0A, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0A">> => Val} | Acc]); +parse_params_(<<16#0B, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0B">> => Val} | Acc]); +parse_params_(<<16#0C, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0C">> => Val} | Acc]); +parse_params_(<<16#0D, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0D">> => Val} | Acc]); +parse_params_(<<16#0E, Rest/binary>>, Acc) -> + case [V || #{<<"0x0D">> := V} <- Acc] of + [Len] -> + <> = Rest, + parse_params_(Rest1, [#{<<"0x0E">> => Val} | Acc]); + _ -> + ?SLOG(error, #{ + msg => "invalid_data", reason => "cmd_0x0D must appear ahead of cmd_0x0E" + }), + lists:reverse(Acc) + end; +parse_params_(<<16#0F, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0F">> => Val} | Acc]); +parse_params_(<<16#10, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x10">> => Val} | Acc]); +parse_params_(Cmd, Acc) -> + ?SLOG(error, #{msg => "unexcepted_param_identifier", cmd => Cmd}), + lists:reverse(Acc). + +parse_ctrl_param(16#01, Param) -> + parse_upgrade_feild(Param); +parse_ctrl_param(16#02, _) -> + <<>>; +parse_ctrl_param(16#03, _) -> + <<>>; +parse_ctrl_param(16#04, _) -> + <<>>; +parse_ctrl_param(16#05, _) -> + <<>>; +parse_ctrl_param(16#06, <>) -> + #{<<"Level">> => Level, <<"Message">> => Msg}; +parse_ctrl_param(16#07, _) -> + <<>>; +parse_ctrl_param(Cmd, Param) -> + ?SLOG(error, #{msg => "unexcepted_param", param => Param, cmd => Cmd}), + <<>>. + +parse_upgrade_feild(Param) -> + [ + DialingName, + Username, + Password, + <<0, 0, I1, I2, I3, I4>>, + <>, + ManufacturerId, + HardwareVer, + SoftwareVer, + UpgradeUrl, + <> + ] = re:split(Param, ";", [{return, binary}]), + + #{ + <<"DialingName">> => DialingName, + <<"Username">> => Username, + <<"Password">> => Password, + <<"Ip">> => list_to_binary(inet:ntoa({I1, I2, I3, I4})), + <<"Port">> => Port, + <<"ManufacturerId">> => ManufacturerId, + <<"HardwareVer">> => HardwareVer, + <<"SoftwareVer">> => SoftwareVer, + <<"UpgradeUrl">> => UpgradeUrl, + <<"Timeout">> => Timeout + }. + +%%-------------------------------------------------------------------- +%% serialize_pkt +%%-------------------------------------------------------------------- +serialize_pkt(Frame, _Opts) -> + serialize(Frame). + +serialize(#frame{cmd = Cmd, ack = Ack, vin = Vin, encrypt = Encrypt, data = Data, rawdata = RawData}) -> + Encrypted = encipher(serialize_data(Cmd, Ack, RawData, Data), Encrypt), + Len = byte_size(Encrypted), + Stream = <>, + Crc = cal_check(Stream, byte_size(Stream), undefined), + <<"##", Stream/binary, Crc:?BYTE>>. + +serialize_data(?CMD_PARAM_QUERY, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Total">> := Total, + <<"Ids">> := Ids +}) when length(Ids) == Total -> + T = tune_time(Time), + Ids1 = tune_ids(Ids), + <>; +serialize_data(?CMD_PARAM_SETTING, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Total">> := Total, + <<"Params">> := Params +}) when length(Params) == Total -> + T = tune_time(Time), + Params1 = tune_params(Params), + <>; +serialize_data(?CMD_TERMINAL_CTRL, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Command">> := Cmd, + <<"Param">> := Param +}) -> + T = tune_time(Time), + Param1 = tune_ctrl_param(Cmd, Param), + <>; +serialize_data(_Cmd, Ack, RawData, #{<<"Time">> := Time}) when ?IS_RESPONSE(Ack) -> + Rest = + case byte_size(RawData) > 6 of + false -> <<>>; + true -> binary:part(RawData, 6, byte_size(RawData) - 6) + end, + T = tune_time(Time), + <>. + +tune_time(#{ + <<"Year">> := Year, + <<"Month">> := Month, + <<"Day">> := Day, + <<"Hour">> := Hour, + <<"Minute">> := Min, + <<"Second">> := Sec +}) -> + <>. + +tune_ids(Ids) -> + lists:foldr( + fun + (Id, Acc) when is_integer(Id) -> + <>; + (Id, Acc) when is_binary(Id) -> + <> + end, + <<>>, + Ids + ). + +tune_params(Params) -> + tune_params_(lists:reverse(Params), <<>>). + +tune_params_([], Bin) -> + Bin; +tune_params_([#{16#01 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#01:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#02 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#02:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#03 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#03:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#04 := Val} | Rest], Bin) -> + {Val_05, Rest1} = take_param(16#05, Rest), + tune_params_(Rest1, <<16#04:?BYTE, Val:?BYTE, 16#05, Val_05:Val/binary, Bin/binary>>); +tune_params_([#{16#05 := Val} | Rest], Bin) -> + tune_params_(Rest ++ [#{16#05 => Val}], Bin); +tune_params_([#{16#06 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#06:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#07 := Val} | Rest], Bin) when byte_size(Val) == 5 -> + tune_params_(Rest, <<16#07:?BYTE, Val/binary, Bin/binary>>); +tune_params_([#{16#08 := Val} | Rest], Bin) when byte_size(Val) == 5 -> + tune_params_(Rest, <<16#08:?BYTE, Val/binary, Bin/binary>>); +tune_params_([#{16#09 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#09:?BYTE, Val:?BYTE, Bin/binary>>); +tune_params_([#{16#0A := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0A:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#0B := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0B:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#0C := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0C:?BYTE, Val:?BYTE, Bin/binary>>); +tune_params_([#{16#0D := Val} | Rest], Bin) -> + {Val_0E, Rest1} = take_param(16#0E, Rest), + tune_params_(Rest1, <<16#0D:?BYTE, Val:?BYTE, 16#0E, Val_0E:Val/binary, Bin/binary>>); +tune_params_([#{16#0E := Val} | Rest], Bin) -> + tune_params_(Rest ++ [#{16#0E => Val}], Bin); +tune_params_([#{16#0F := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0F:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#10 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#10:?BYTE, Val:?BYTE, Bin/binary>>). + +tune_ctrl_param(16#00, _) -> + <<>>; +tune_ctrl_param(16#01, Param) -> + tune_upgrade_feild(Param); +tune_ctrl_param(16#02, _) -> + <<>>; +tune_ctrl_param(16#03, _) -> + <<>>; +tune_ctrl_param(16#04, _) -> + <<>>; +tune_ctrl_param(16#05, _) -> + <<>>; +tune_ctrl_param(16#06, #{<<"Level">> := Level, <<"Message">> := Msg}) -> + <>; +tune_ctrl_param(16#07, _) -> + <<>>; +tune_ctrl_param(Cmd, Param) -> + ?SLOG(error, #{msg => "unexcepted_cmd", cmd => Cmd, param => Param}), + <<>>. + +tune_upgrade_feild(Param) -> + TuneBin = fun + (Bin, Len) when is_binary(Bin), byte_size(Bin) =:= Len -> Bin; + (undefined, _) -> undefined; + (Bin, _) -> error({invalid_param_length, Bin}) + end, + TuneWrd = fun + (Val) when is_integer(Val), Val < 65535 -> <>; + (undefined) -> undefined; + (_) -> error(invalid_param_word_value) + end, + TuneAdr = fun + (Ip) when is_binary(Ip) -> + {ok, {I1, I2, I3, I4}} = inet:parse_address(binary_to_list(Ip)), + <<0, 0, I1, I2, I3, I4>>; + (undefined) -> + undefined; + (_) -> + error(invalid_ip_address) + end, + L = [ + maps:get(<<"DialingName">>, Param, undefined), + maps:get(<<"Username">>, Param, undefined), + maps:get(<<"Password">>, Param, undefined), + TuneAdr(maps:get(<<"Ip">>, Param, undefined)), + TuneWrd(maps:get(<<"Port">>, Param, undefined)), + TuneBin(maps:get(<<"ManufacturerId">>, Param, undefined), 4), + TuneBin(maps:get(<<"HardwareVer">>, Param, undefined), 5), + TuneBin(maps:get(<<"SoftwareVer">>, Param, undefined), 5), + maps:get(<<"UpgradeUrl">>, Param, undefined), + TuneWrd(maps:get(<<"Timeout">>, Param, undefined)) + ], + list_to_binary([I || I <- lists:join(";", L), I /= undefined]). + +take_param(K, Params) -> + V = search_param(K, Params), + {V, Params -- [#{K => V}]}. + +search_param(16#05, [#{16#05 := V} | _]) -> V; +search_param(16#0E, [#{16#0E := V} | _]) -> V; +search_param(K, [_ | Rest]) -> search_param(K, Rest). + +cal_check(_, 0, Check) -> Check; +cal_check(<>, Size, undefined) -> cal_check(Rest, Size - 1, C); +cal_check(<>, Size, Check) -> cal_check(Rest, Size - 1, Check bxor C). + +format(Msg) -> + io_lib:format("~p", [Msg]). + +type(_) -> + gbt32960. + +is_message(#frame{}) -> + true; +is_message(_) -> + false. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl new file mode 100644 index 000000000..bc86b9686 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(gbt32960) -> + [ + {mountpoint, emqx_gateway_schema:mountpoint()}, + {retry_interval, + sc( + emqx_schema:duration_ms(), + #{ + default => <<"8s">>, + desc => ?DESC(retry_interval) + } + )}, + {max_retry_times, + sc( + non_neg_integer(), + #{ + default => 3, + desc => ?DESC(max_retry_times) + } + )}, + {message_queue_len, + sc( + non_neg_integer(), + #{ + default => 10, + desc => ?DESC(message_queue_len) + } + )}, + {listeners, sc(ref(emqx_gateway_schema, tcp_listeners), #{desc => ?DESC(tcp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(). + +desc(gbt32960) -> + "The GBT-32960 gateway"; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 5ea6eee70..4292ce50d 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -65,6 +65,7 @@ emqx_gateway_coap, emqx_gateway_lwm2m, emqx_gateway_exproto, + emqx_gateway_gbt32960, emqx_exhook, emqx_bridge, emqx_bridge_mqtt, diff --git a/mix.exs b/mix.exs index ed869c414..11e0099bd 100644 --- a/mix.exs +++ b/mix.exs @@ -329,6 +329,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_gateway_coap, :emqx_gateway_lwm2m, :emqx_gateway_exproto, + :emqx_gateway_gbt32960, :emqx_dashboard, :emqx_dashboard_sso, :emqx_audit, diff --git a/rel/i18n/emqx_gateway_api.hocon b/rel/i18n/emqx_gateway_api.hocon index 1e0e22456..d712a054d 100644 --- a/rel/i18n/emqx_gateway_api.hocon +++ b/rel/i18n/emqx_gateway_api.hocon @@ -38,7 +38,7 @@ gateway_name.desc: gateway_name_in_qs.desc: """Gateway Name.
-It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`""" +It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`, `gbt32960`""" gateway_node_status.desc: """The status of the gateway on each node in the cluster""" diff --git a/rel/i18n/emqx_gbt32960_schema.hocon b/rel/i18n/emqx_gbt32960_schema.hocon new file mode 100644 index 000000000..0827a05c4 --- /dev/null +++ b/rel/i18n/emqx_gbt32960_schema.hocon @@ -0,0 +1,12 @@ +emqx_gbt32960_schema { + +retry_interval.desc: +"""Re-send time interval""" + +max_retry_times.desc: +"""Re-send max times""" + +message_queue_len.desc: +"""Max message queue length""" + +} From b35cd57034103ba454a193795671587e512b6f8b Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 31 Oct 2023 19:04:21 +0800 Subject: [PATCH 043/111] fix(gbt32960): make elvis happy --- .../src/emqx_gbt32960_frame.erl | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl index a5ec3fa77..641f2c02d 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl @@ -33,7 +33,7 @@ Ack == ?ACK_VIN_REPEAT ). --type phase() :: search_heading_0x23 | parse. +-type phase() :: search_heading | parse. -type parser_state() :: #{ data := binary(), @@ -46,7 +46,7 @@ -spec initial_parse_state(map()) -> parser_state(). initial_parse_state(_) -> - #{data => <<>>, phase => search_heading_0x23}. + #{data => <<>>, phase => search_heading}. -spec serialize_opts() -> emqx_gateway_frame:serialize_options(). serialize_opts() -> @@ -58,15 +58,15 @@ serialize_opts() -> parse(Bin, State) -> case enter_parse(Bin, State) of {ok, Message, Rest} -> - {ok, Message, Rest, State#{parse => search_heading_0x23}}; + {ok, Message, Rest, State#{parse => search_heading}}; {error, Error} -> {error, Error}; {more_data_follow, Partial} -> {more, State#{data => Partial, phase => parse}} end. -enter_parse(Bin, #{phase := search_heading_0x23}) -> - case search_heading_0x23(Bin) of +enter_parse(Bin, #{phase := search_heading}) -> + case search_heading(Bin) of {ok, Rest} -> parse_msg(Rest); Error -> @@ -75,11 +75,11 @@ enter_parse(Bin, #{phase := search_heading_0x23}) -> enter_parse(Bin, #{data := Data}) -> parse_msg(<>). -search_heading_0x23(<<16#23, 16#23, Rest/binary>>) -> +search_heading(<<16#23, 16#23, Rest/binary>>) -> {ok, Rest}; -search_heading_0x23(<<_, Rest/binary>>) -> - search_heading_0x23(Rest); -search_heading_0x23(<<>>) -> +search_heading(<<_, Rest/binary>>) -> + search_heading(Rest); +search_heading(<<>>) -> {error, invalid_frame}. parse_msg(Binary) -> @@ -329,8 +329,8 @@ parse_info(<>, Acc) -> <> = Rest1, - <> = Rest2, + <> = Rest2, parse_info(Rest3, [ #{ <<"Type">> => <<"FuelCell">>, @@ -339,12 +339,12 @@ parse_info(<>, Acc) -> <<"FuelConsumption">> => FuelConsumption, <<"ProbeNum">> => ProbeNum, <<"ProbeTemps">> => binary_to_list(ProbeTemps), - <<"H_MaxTemp">> => H_MaxTemp, - <<"H_TempProbeCode">> => H_TempProbeCode, - <<"H_MaxConc">> => H_MaxConc, - <<"H_ConcSensorCode">> => H_ConcSensorCode, - <<"H_MaxPress">> => H_MaxPress, - <<"H_PressSensorCode">> => H_PressSensorCode, + <<"H_MaxTemp">> => HMaxTemp, + <<"H_TempProbeCode">> => HTempProbeCode, + <<"H_MaxConc">> => HMaxConc, + <<"H_ConcSensorCode">> => HConcSensorCode, + <<"H_MaxPress">> => HMaxPress, + <<"H_PressSensorCode">> => HPressSensorCode, <<"DCStatus">> => DCStatus } | Acc @@ -696,8 +696,8 @@ tune_params_([#{16#02 := Val} | Rest], Bin) -> tune_params_([#{16#03 := Val} | Rest], Bin) -> tune_params_(Rest, <<16#03:?BYTE, Val:?WORD, Bin/binary>>); tune_params_([#{16#04 := Val} | Rest], Bin) -> - {Val_05, Rest1} = take_param(16#05, Rest), - tune_params_(Rest1, <<16#04:?BYTE, Val:?BYTE, 16#05, Val_05:Val/binary, Bin/binary>>); + {Val05, Rest1} = take_param(16#05, Rest), + tune_params_(Rest1, <<16#04:?BYTE, Val:?BYTE, 16#05, Val05:Val/binary, Bin/binary>>); tune_params_([#{16#05 := Val} | Rest], Bin) -> tune_params_(Rest ++ [#{16#05 => Val}], Bin); tune_params_([#{16#06 := Val} | Rest], Bin) -> @@ -715,8 +715,8 @@ tune_params_([#{16#0B := Val} | Rest], Bin) -> tune_params_([#{16#0C := Val} | Rest], Bin) -> tune_params_(Rest, <<16#0C:?BYTE, Val:?BYTE, Bin/binary>>); tune_params_([#{16#0D := Val} | Rest], Bin) -> - {Val_0E, Rest1} = take_param(16#0E, Rest), - tune_params_(Rest1, <<16#0D:?BYTE, Val:?BYTE, 16#0E, Val_0E:Val/binary, Bin/binary>>); + {Val0E, Rest1} = take_param(16#0E, Rest), + tune_params_(Rest1, <<16#0D:?BYTE, Val:?BYTE, 16#0E, Val0E:Val/binary, Bin/binary>>); tune_params_([#{16#0E := Val} | Rest], Bin) -> tune_params_(Rest ++ [#{16#0E => Val}], Bin); tune_params_([#{16#0F := Val} | Rest], Bin) -> From d278486416d002d4ecc34f160c04c26c2c03f8d8 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 30 Oct 2023 21:48:42 +0700 Subject: [PATCH 044/111] fix(secret): dedicate a specific loader module for file secrets To make code employing `emqx_secret` easier to follow. --- apps/emqx/src/emqx_schema_secret.erl | 33 +++++--------------- apps/emqx/src/emqx_secret.erl | 14 +++++---- apps/emqx/src/emqx_secret_loader.erl | 42 ++++++++++++++++++++++++++ apps/emqx/test/emqx_secret_tests.erl | 45 ++++++++++++++++------------ 4 files changed, 84 insertions(+), 50 deletions(-) create mode 100644 apps/emqx/src/emqx_secret_loader.erl diff --git a/apps/emqx/src/emqx_schema_secret.erl b/apps/emqx/src/emqx_schema_secret.erl index 865d0eac9..635285ce7 100644 --- a/apps/emqx/src/emqx_schema_secret.erl +++ b/apps/emqx/src/emqx_schema_secret.erl @@ -25,9 +25,6 @@ %% HOCON Schema API -export([convert_secret/2]). -%% Target of `emqx_secret:wrap/3` --export([load/1]). - %% @doc Secret value. -type t() :: binary(). @@ -74,31 +71,15 @@ convert_secret(Secret, #{}) -> end. -spec wrap(source()) -> emqx_secret:t(t()). -wrap(Source) -> - emqx_secret:wrap(?MODULE, load, Source). +wrap(<<"file://", Filename/binary>>) -> + emqx_secret:wrap_load({file, Filename}); +wrap(Secret) -> + emqx_secret:wrap(Secret). -spec source(emqx_secret:t(t())) -> source(). source(Secret) when is_function(Secret) -> - emqx_secret:term(Secret); + source(emqx_secret:term(Secret)); +source({file, Filename}) -> + <<"file://", Filename/binary>>; source(Secret) -> Secret. - -%% - --spec load(source()) -> t(). -load(<<"file://", Filename/binary>>) -> - load_file(Filename); -load(Secret) -> - Secret. - -load_file(Filename) -> - case file:read_file(Filename) of - {ok, Secret} -> - string:trim(Secret, trailing, [$\n]); - {error, Reason} -> - throw(#{ - msg => failed_to_read_secret_file, - path => Filename, - reason => emqx_utils:explain_posix(Reason) - }) - end. diff --git a/apps/emqx/src/emqx_secret.erl b/apps/emqx/src/emqx_secret.erl index ad0194201..dfbfa488e 100644 --- a/apps/emqx/src/emqx_secret.erl +++ b/apps/emqx/src/emqx_secret.erl @@ -19,12 +19,16 @@ -module(emqx_secret). %% API: --export([wrap/1, wrap/3, unwrap/1, term/1]). +-export([wrap/1, wrap_load/1, unwrap/1, term/1]). -export_type([t/1]). -opaque t(T) :: T | fun(() -> t(T)). +%% Secret loader module. +%% Any changes related to processing of secrets should be made there. +-define(LOADER, emqx_secret_loader). + %%================================================================================ %% API funcions %%================================================================================ @@ -37,12 +41,12 @@ wrap(Term) -> Term end. -%% @doc Wrap a function call over a term in a secret closure. +%% @doc Wrap a loader function call over a term in a secret closure. %% This is slightly more flexible form of `wrap/1` with the same basic purpose. --spec wrap(module(), atom(), _Term) -> t(_). -wrap(Module, Function, Term) -> +-spec wrap_load(emqx_secret_loader:source()) -> t(_). +wrap_load(Source) -> fun() -> - apply(Module, Function, [Term]) + apply(?LOADER, load, [Source]) end. %% @doc Unwrap a secret closure, revealing the secret. diff --git a/apps/emqx/src/emqx_secret_loader.erl b/apps/emqx/src/emqx_secret_loader.erl new file mode 100644 index 000000000..2e99587bf --- /dev/null +++ b/apps/emqx/src/emqx_secret_loader.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_secret_loader). + +%% API +-export([load/1]). +-export([file/1]). + +-export_type([source/0]). + +-type source() :: {file, file:filename_all()}. + +-spec load(source()) -> binary() | no_return(). +load({file, Filename}) -> + file(Filename). + +-spec file(file:filename_all()) -> binary() | no_return(). +file(Filename) -> + case file:read_file(Filename) of + {ok, Secret} -> + string:trim(Secret, trailing); + {error, Reason} -> + throw(#{ + msg => failed_to_read_secret_file, + path => Filename, + reason => emqx_utils:explain_posix(Reason) + }) + end. diff --git a/apps/emqx/test/emqx_secret_tests.erl b/apps/emqx/test/emqx_secret_tests.erl index ab0866c10..cd6588c83 100644 --- a/apps/emqx/test/emqx_secret_tests.erl +++ b/apps/emqx/test/emqx_secret_tests.erl @@ -16,8 +16,6 @@ -module(emqx_secret_tests). --export([ident/1]). - -include_lib("eunit/include/eunit.hrl"). wrap_unwrap_test() -> @@ -32,16 +30,30 @@ unwrap_immediate_test() -> emqx_secret:unwrap(42) ). -wrap_unwrap_external_test() -> +wrap_unwrap_load_test_() -> + Secret = <<"foobaz">>, + { + setup, + fun() -> write_temp_file(Secret) end, + fun(Filename) -> file:delete(Filename) end, + fun(Filename) -> + ?_assertEqual( + Secret, + emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename})) + ) + end + }. + +wrap_load_term_test() -> ?assertEqual( - ident({foo, bar}), - emqx_secret:unwrap(emqx_secret:wrap(?MODULE, ident, {foo, bar})) + {file, "no/such/file/i/swear"}, + emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"})) ). -wrap_unwrap_transform_test() -> - ?assertEqual( - <<"this_was_an_atom">>, - emqx_secret:unwrap(emqx_secret:wrap(erlang, atom_to_binary, this_was_an_atom)) +wrap_unwrap_missing_file_test() -> + ?assertThrow( + #{msg := failed_to_read_secret_file, reason := "No such file or directory"}, + emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"})) ). wrap_term_test() -> @@ -50,12 +62,6 @@ wrap_term_test() -> emqx_secret:term(emqx_secret:wrap(42)) ). -wrap_external_term_test() -> - ?assertEqual( - this_was_an_atom, - emqx_secret:term(emqx_secret:wrap(erlang, atom_to_binary, this_was_an_atom)) - ). - external_fun_term_error_test() -> Term = {foo, bar}, ?assertError( @@ -63,7 +69,8 @@ external_fun_term_error_test() -> emqx_secret:term(fun() -> Term end) ). -%% - -ident(X) -> - X. +write_temp_file(Bytes) -> + Ts = erlang:system_time(millisecond), + Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)), + ok = file:write_file(Filename, Bytes), + Filename. From 5e314d4ef1b2b1d9dddedb65cda60a04e50b93e2 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 1 Nov 2023 13:17:01 +0800 Subject: [PATCH 045/111] test(gbt32960): add test suites --- .../src/emqx_gbt32960_channel.erl | 7 +- .../src/emqx_gbt32960_frame.erl | 4 + .../src/emqx_gbt32960_schema.erl | 4 +- .../test/emqx_gbt32960_SUITE.erl | 1444 +++++++++++++++++ .../test/emqx_gbt32960_parser_SUITE.erl | 924 +++++++++++ 5 files changed, 2381 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl create mode 100644 apps/emqx_gateway_gbt32960/test/emqx_gbt32960_parser_SUITE.erl diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl index 6b491a807..b5ac046ce 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl @@ -92,6 +92,8 @@ info(Channel) -> -spec info(list(atom()) | atom(), channel()) -> term(). info(Keys, Channel) when is_list(Keys) -> [{Key, info(Key, Channel)} || Key <- Keys]; +info(ctx, #channel{ctx = Ctx}) -> + Ctx; info(conninfo, #channel{conninfo = ConnInfo}) -> ConnInfo; info(zone, #channel{clientinfo = #{zone := Zone}}) -> @@ -279,6 +281,9 @@ handle_out({AckCode, Frame}, Channel) when -> {ok, [{outgoing, ack(AckCode, Frame)}], Channel}. +handle_out({AckCode, Frame}, Outgoings, Channel) when ?IS_ACK_CODE(AckCode) -> + {ok, [{outgoing, ack(AckCode, Frame)} | Outgoings], Channel}. + %%-------------------------------------------------------------------- %% Handle Delivers from broker to client %%-------------------------------------------------------------------- @@ -609,7 +614,7 @@ process_connect( _ = upstreaming(Frame, NChannel), %% XXX: connection_accepted is not defined by stomp protocol _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, #{}]), - {ok, NChannel}; + handle_out({?ACK_SUCCESS, Frame}, [{event, connected}], NChannel); {error, Reason} -> log( error, diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl index 641f2c02d..8fd95a6b5 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl @@ -40,6 +40,10 @@ phase := phase() }. +-ifdef(TEST). +-export([serialize/1]). +-endif. + %%-------------------------------------------------------------------- %% Init a Parser %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl index bc86b9686..abc1623f6 100644 --- a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl @@ -7,12 +7,14 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). +-define(DEFAULT_MOUNTPOINT, <<"gbt32960/${clientid}">>). + %% config schema provides -export([fields/1, desc/1]). fields(gbt32960) -> [ - {mountpoint, emqx_gateway_schema:mountpoint()}, + {mountpoint, emqx_gateway_schema:mountpoint(?DEFAULT_MOUNTPOINT)}, {retry_interval, sc( emqx_schema:duration_ms(), diff --git a/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl new file mode 100644 index 000000000..16d56e28a --- /dev/null +++ b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl @@ -0,0 +1,1444 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BYTE, 8 / big - integer). +-define(WORD, 16 / big - integer). +-define(DWORD, 32 / big - integer). + +-define(PORT, 7325). +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). + +-define(CONF_DEFAULT, << + "\n" + "gateway.gbt32960 {\n" + " retry_interval = \"1s\"\n" + " listeners.tcp.default {\n" + " bind = 7325\n" + " }\n" + "}\n" +>>). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_gateway_gbt32960), + Apps = emqx_cth_suite:start( + [ + {emqx_conf, ?CONF_DEFAULT}, + emqx_gateway, + emqx_auth, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + emqx_common_test_http:create_default_app(), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + emqx_common_test_http:delete_default_app(), + emqx_cth_suite:stop(?config(suite_apps, Config)), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +encode(Cmd, Vin, Data) -> + encode(Cmd, ?ACK_IS_CMD, Vin, ?ENCRYPT_NONE, Data). + +encode(Cmd, Ack, Vin, Data) -> + encode(Cmd, Ack, Vin, ?ENCRYPT_NONE, Data). + +encode(Cmd, Ack, Vin, Encrypt, Data) -> + Size = byte_size(Data), + S1 = <>, + Crc = make_crc(S1, undefined), + Stream = <<"##", S1/binary, Crc:8>>, + ?LOGT("encode a packet=~p", [binary_to_hex_string(Stream)]), + Stream. + +make_crc(<<>>, Xor) -> Xor; +make_crc(<>, undefined) -> make_crc(Rest, C); +make_crc(<>, Xor) -> make_crc(Rest, C bxor Xor). + +make_time() -> + {Year, Mon, Day} = date(), + {Hour, Min, Sec} = time(), + Year1 = list_to_integer(string:substr(integer_to_list(Year), 3, 2)), + <>. + +binary_to_hex_string(Data) -> + lists:flatten([io_lib:format("~2.16.0B ", [X]) || <> <= Data]). + +to_json(#frame{cmd = Cmd, vin = Vin, encrypt = Encrypt, data = Data}) -> + emqx_utils_json:encode(#{'Cmd' => Cmd, 'Vin' => Vin, 'Encrypt' => Encrypt, 'Data' => Data}). + +get_published_msg() -> + receive + {deliver, _Topic, #message{topic = Topic, payload = Payload}} -> + {Topic, Payload} + after 5000 -> + error(timeout) + end. + +get_subscriptions() -> + lists:map(fun({_, Topic}) -> Topic end, ets:tab2list(emqx_subscription)). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% test cases %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +login_first() -> + emqx:subscribe("gbt32960/+/upstream/#"), + + % + % send VEHICLE LOGIN + % + Time = <<12, 12, 29, 12, 19, 20>>, + Data = <