From 6354f3b04ff2d0dd1dee7512997782d08203357d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 16 Oct 2023 23:57:23 +0300 Subject: [PATCH 1/7] feat(authn): allow authn providers to define a separate schama for API --- .../src/emqx_authn/emqx_authn_api.erl | 23 ++-- .../emqx_authn_password_hashing.erl | 100 +++++++++++------- .../src/emqx_authn/emqx_authn_schema.erl | 68 +++++++++--- .../test/emqx_authn/emqx_authn_api_SUITE.erl | 36 ++++++- .../test/emqx_authz/emqx_authz_SUITE.erl | 1 + .../src/emqx_authn_mnesia_schema.erl | 30 ++++-- apps/emqx_utils/src/emqx_utils.erl | 19 +++- changes/ce/fix-11771.en.md | 1 + 8 files changed, 202 insertions(+), 76 deletions(-) create mode 100644 changes/ce/fix-11771.en.md diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl index 9938a3018..f30f7f473 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl @@ -147,7 +147,7 @@ schema("/authentication") -> description => ?DESC(authentication_get), responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - hoconsc:array(emqx_authn_schema:authenticator_type()), + hoconsc:array(authenticator_type(config)), authenticator_array_example() ) } @@ -156,12 +156,12 @@ schema("/authentication") -> tags => ?API_TAGS_GLOBAL, description => ?DESC(authentication_post), 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -178,7 +178,7 @@ schema("/authentication/:id") -> parameters => [param_auth_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) @@ -189,7 +189,7 @@ schema("/authentication/:id") -> description => ?DESC(authentication_id_put), parameters => [param_auth_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ @@ -236,7 +236,7 @@ schema("/listeners/:listener_id/authentication") -> parameters => [param_listener_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_example( - hoconsc:array(emqx_authn_schema:authenticator_type()), + hoconsc:array(authenticator_type(config)), authenticator_array_example() ) } @@ -247,12 +247,12 @@ schema("/listeners/:listener_id/authentication") -> description => ?DESC(listeners_listener_id_authentication_post), parameters => [param_listener_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>), @@ -270,7 +270,7 @@ schema("/listeners/:listener_id/authentication/:id") -> parameters => [param_listener_id(), param_auth_id()], responses => #{ 200 => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(config), authenticator_examples() ), 404 => error_codes([?NOT_FOUND], <<"Not Found">>) @@ -282,7 +282,7 @@ schema("/listeners/:listener_id/authentication/:id") -> description => ?DESC(listeners_listener_id_authentication_id_put), parameters => [param_listener_id(), param_auth_id()], 'requestBody' => emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type(), + authenticator_type(api_write), authenticator_examples() ), responses => #{ @@ -1278,6 +1278,9 @@ paginated_list_type(Type) -> {meta, ref(emqx_dashboard_swagger, meta)} ]. +authenticator_type(Kind) -> + emqx_authn_schema:authenticator_type(Kind). + authenticator_array_example() -> [Config || #{value := Config} <- maps:values(authenticator_examples())]. diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl index 66bc6bfc6..40e96ce6f 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl @@ -53,7 +53,8 @@ -export([ type_ro/1, - type_rw/1 + type_rw/1, + type_rw_api/1 ]). -export([ @@ -67,21 +68,17 @@ -define(SALT_ROUNDS_MAX, 10). namespace() -> "authn-hash". -roots() -> [pbkdf2, bcrypt, bcrypt_rw, simple]. +roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple]. fields(bcrypt_rw) -> fields(bcrypt) ++ [ - {salt_rounds, - sc( - range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX), - #{ - default => ?SALT_ROUNDS_MAX, - example => ?SALT_ROUNDS_MAX, - desc => "Work factor for BCRYPT password generation.", - converter => fun salt_rounds_converter/2 - } - )} + {salt_rounds, fun bcrypt_salt_rounds/1} + ]; +fields(bcrypt_rw_api) -> + fields(bcrypt) ++ + [ + {salt_rounds, fun bcrypt_salt_rounds_api/1} ]; fields(bcrypt) -> [{name, sc(bcrypt, #{required => true, desc => "BCRYPT password hashing."})}]; @@ -110,6 +107,15 @@ fields(simple) -> {salt_position, fun salt_position/1} ]. +bcrypt_salt_rounds(converter) -> fun salt_rounds_converter/2; +bcrypt_salt_rounds(Option) -> bcrypt_salt_rounds_api(Option). + +bcrypt_salt_rounds_api(type) -> range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX); +bcrypt_salt_rounds_api(default) -> ?SALT_ROUNDS_MAX; +bcrypt_salt_rounds_api(example) -> ?SALT_ROUNDS_MAX; +bcrypt_salt_rounds_api(desc) -> "Work factor for BCRYPT password generation."; +bcrypt_salt_rounds_api(_) -> undefined. + salt_rounds_converter(undefined, _) -> undefined; salt_rounds_converter(I, _) when is_integer(I) -> @@ -119,6 +125,8 @@ salt_rounds_converter(X, _) -> desc(bcrypt_rw) -> "Settings for bcrypt password hashing algorithm (for DB backends with write capability)."; +desc(bcrypt_rw_api) -> + desc(bcrypt_rw); desc(bcrypt) -> "Settings for bcrypt password hashing algorithm."; desc(pbkdf2) -> @@ -143,14 +151,20 @@ dk_length(desc) -> dk_length(_) -> undefined. -%% for simple_authn/emqx_authn_mnesia +%% for emqx_authn_mnesia type_rw(type) -> hoconsc:union(rw_refs()); -type_rw(default) -> - #{<<"name">> => sha256, <<"salt_position">> => prefix}; type_rw(desc) -> "Options for password hash creation and verification."; -type_rw(_) -> +type_rw(Option) -> + type_ro(Option). + +%% for emqx_authn_mnesia API +type_rw_api(type) -> + hoconsc:union(api_refs()); +type_rw_api(desc) -> + "Options for password hash creation and verification through API."; +type_rw_api(_) -> undefined. %% for other authn resources @@ -242,31 +256,41 @@ check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHa %%------------------------------------------------------------------------------ rw_refs() -> - All = [ - hoconsc:ref(?MODULE, bcrypt_rw), - hoconsc:ref(?MODULE, pbkdf2), - hoconsc:ref(?MODULE, simple) - ], - fun - (all_union_members) -> All; - ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt_rw)]; - ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)]; - ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)]; - ({value, _}) -> throw(#{reason => "algorithm_name_missing"}) - end. + union_selector(rw). ro_refs() -> - All = [ - hoconsc:ref(?MODULE, bcrypt), - hoconsc:ref(?MODULE, pbkdf2), - hoconsc:ref(?MODULE, simple) - ], + union_selector(ro). + +api_refs() -> + union_selector(api). + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). + +union_selector(Kind) -> fun - (all_union_members) -> All; - ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt)]; - ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)]; - ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)]; + (all_union_members) -> refs(Kind); + ({value, #{<<"name">> := <<"bcrypt">>}}) -> [bcrypt_ref(Kind)]; + ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [pbkdf2_ref(Kind)]; + ({value, #{<<"name">> := _}}) -> [simple_ref(Kind)]; ({value, _}) -> throw(#{reason => "algorithm_name_missing"}) end. -sc(Type, Meta) -> hoconsc:mk(Type, Meta). +refs(Kind) -> + [ + bcrypt_ref(Kind), + pbkdf2_ref(Kind), + simple_ref(Kind) + ]. + +pbkdf2_ref(_) -> + hoconsc:ref(?MODULE, pbkdf2). + +bcrypt_ref(rw) -> + hoconsc:ref(?MODULE, bcrypt_rw); +bcrypt_ref(api) -> + hoconsc:ref(?MODULE, bcrypt_rw_api); +bcrypt_ref(_) -> + hoconsc:ref(?MODULE, bcrypt). + +simple_ref(_) -> + hoconsc:ref(?MODULE, simple). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index a06d4b692..9b9935a1f 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -34,7 +34,9 @@ tags/0, fields/1, authenticator_type/0, + authenticator_type/1, authenticator_type_without/1, + authenticator_type_without/2, mechanism/1, backend/1 ]). @@ -43,17 +45,35 @@ global_auth_fields/0 ]). +-export_type([shema_kind/0]). + -define(AUTHN_MODS_PT_KEY, {?MODULE, authn_schema_mods}). +-define(DEFAULT_SCHEMA_KIND, config). %%-------------------------------------------------------------------- %% Authn Source Schema Behaviour %%-------------------------------------------------------------------- -type schema_ref() :: ?R_REF(module(), hocon_schema:name()). +-type shema_kind() :: + %% api_write: schema for mutating API request validation + api_write + %% config: schema for config validation + | config. -callback refs() -> [schema_ref()]. --callback select_union_member(emqx_config:raw_config()) -> schema_ref() | undefined | no_return(). +-callback refs(shema_kind()) -> [schema_ref()]. +-callback select_union_member(emqx_config:raw_config()) -> [schema_ref()] | undefined | no_return(). +-callback select_union_member(shema_kind(), emqx_config:raw_config()) -> + [schema_ref()] | undefined | no_return(). -callback fields(hocon_schema:name()) -> [hocon_schema:field()]. +-optional_callbacks([ + select_union_member/1, + select_union_member/2, + refs/0, + refs/1 +]). + roots() -> []. injected_fields(AuthnSchemaMods) -> @@ -67,45 +87,63 @@ tags() -> [<<"Authentication">>]. authenticator_type() -> - hoconsc:union(union_member_selector(provider_schema_mods())). + authenticator_type(?DEFAULT_SCHEMA_KIND). + +authenticator_type(Kind) -> + hoconsc:union(union_member_selector(Kind, provider_schema_mods())). authenticator_type_without(ProviderSchemaMods) -> + authenticator_type_without(?DEFAULT_SCHEMA_KIND, ProviderSchemaMods). + +authenticator_type_without(Kind, ProviderSchemaMods) -> hoconsc:union( - union_member_selector(provider_schema_mods() -- ProviderSchemaMods) + union_member_selector(Kind, provider_schema_mods() -- ProviderSchemaMods) ). -union_member_selector(Mods) -> - AllTypes = config_refs(Mods), +union_member_selector(Kind, Mods) -> + AllTypes = config_refs(Kind, Mods), fun (all_union_members) -> AllTypes; - ({value, Value}) -> select_union_member(Value, Mods) + ({value, Value}) -> select_union_member(Kind, Value, Mods) end. -select_union_member(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) -> +select_union_member(_Kind, #{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) -> throw(#{ reason => "unsupported_mechanism", mechanism => Mechanism, backend => Backend }); -select_union_member(#{<<"mechanism">> := Mechanism}, []) -> +select_union_member(_Kind, #{<<"mechanism">> := Mechanism}, []) -> throw(#{ reason => "unsupported_mechanism", mechanism => Mechanism }); -select_union_member(#{<<"mechanism">> := _} = Value, [Mod | Mods]) -> - case Mod:select_union_member(Value) of +select_union_member(Kind, #{<<"mechanism">> := _} = Value, [Mod | Mods]) -> + case mod_select_union_member(Kind, Value, Mod) of undefined -> - select_union_member(Value, Mods); + select_union_member(Kind, Value, Mods); Member -> Member end; -select_union_member(#{} = _Value, _Mods) -> +select_union_member(_Kind, #{} = _Value, _Mods) -> throw(#{reason => "missing_mechanism_field"}); -select_union_member(Value, _Mods) -> +select_union_member(_Kind, Value, _Mods) -> throw(#{reason => "not_a_struct", value => Value}). -config_refs(Mods) -> - lists:append([Mod:refs() || Mod <- Mods]). +mod_select_union_member(Kind, Value, Mod) -> + emqx_utils:call_first_defined([ + {Mod, select_union_member, [Kind, Value]}, + {Mod, select_union_member, [Value]} + ]). + +config_refs(Kind, Mods) -> + lists:append([mod_refs(Kind, Mod) || Mod <- Mods]). + +mod_refs(Kind, Mod) -> + emqx_utils:call_first_defined([ + {Mod, refs, [Kind]}, + {Mod, refs, []} + ]). root_type() -> hoconsc:array(authenticator_type()). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl index 635b157d9..45a605e6e 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl @@ -63,14 +63,16 @@ end_per_testcase(_, Config) -> init_per_suite(Config) -> Apps = emqx_cth_suite:start( [ - emqx, emqx_conf, + emqx, emqx_auth, + %% to load schema + {emqx_auth_mnesia, #{start => false}}, emqx_management, {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ], #{ - work_dir => ?config(priv_dir, Config) + work_dir => filename:join(?config(priv_dir, Config), ?MODULE) } ), _ = emqx_common_test_http:create_default_app(), @@ -535,6 +537,36 @@ ignore_switch_to_global_chain(_) -> ), ok = emqtt:disconnect(Client4). +t_bcrypt_validation(_Config) -> + BaseConf = #{ + mechanism => <<"password_based">>, + backend => <<"built_in_database">>, + user_id_type => <<"username">> + }, + BcryptValid = #{ + name => <<"bcrypt">>, + salt_rounds => 10 + }, + BcryptInvalid = #{ + name => <<"bcrypt">>, + salt_rounds => 15 + }, + + ConfValid = BaseConf#{password_hash_algorithm => BcryptValid}, + ConfInvalid = BaseConf#{password_hash_algorithm => BcryptInvalid}, + + {ok, 400, _} = request( + post, + uri([?CONF_NS]), + ConfInvalid + ), + + {ok, 200, _} = request( + post, + uri([?CONF_NS]), + ConfValid + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl index 1af7d4d1d..37c9ebfc1 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl @@ -70,6 +70,7 @@ init_per_testcase(TestCase, Config) when {ok, _} = emqx:update_config([authorization, deny_action], disconnect), Config; init_per_testcase(_TestCase, Config) -> + _ = file:delete(emqx_authz_file:acl_conf_file()), {ok, _} = emqx_authz:update(?CMD_REPLACE, []), Config. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl index 2d57abc90..bb5ccfe1a 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl @@ -24,27 +24,30 @@ -export([ fields/1, desc/1, - refs/0, - select_union_member/1 + refs/1, + select_union_member/2 ]). -refs() -> +refs(api_write) -> + [?R_REF(builtin_db_api)]; +refs(_) -> [?R_REF(builtin_db)]. -select_union_member(#{ +select_union_member(Kind, #{ <<"mechanism">> := ?AUTHN_MECHANISM_SIMPLE_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN }) -> - refs(); -select_union_member(_) -> + refs(Kind); +select_union_member(_Kind, _Value) -> undefined. fields(builtin_db) -> [ - {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)}, - {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, - {user_id_type, fun user_id_type/1}, {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} - ] ++ emqx_authn_schema:common_fields(). + ] ++ common_fields(); +fields(builtin_db_api) -> + [ + {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1} + ] ++ common_fields(). desc(builtin_db) -> ?DESC(builtin_db); @@ -56,3 +59,10 @@ user_id_type(desc) -> ?DESC(?FUNCTION_NAME); user_id_type(default) -> <<"username">>; user_id_type(required) -> true; user_id_type(_) -> undefined. + +common_fields() -> + [ + {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)}, + {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, + {user_id_type, fun user_id_type/1} + ] ++ emqx_authn_schema:common_fields(). diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index bf4e07ff9..f827f65de 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -62,7 +62,8 @@ merge_lists/3, tcp_keepalive_opts/4, format/1, - format_mfal/1 + format_mfal/1, + call_first_defined/1 ]). -export([ @@ -554,6 +555,22 @@ format_mfal(Data) -> undefined end. +-spec call_first_defined(list({module(), atom(), list()})) -> term() | no_return(). +call_first_defined([{Module, Function, Args} | Rest]) -> + try + apply(Module, Function, Args) + catch + error:undef:Stacktrace -> + case Stacktrace of + [{Module, Function, _, _} | _] -> + call_first_defined(Rest); + _ -> + erlang:raise(error, undef, Stacktrace) + end + end; +call_first_defined([]) -> + error(none_fun_is_defined). + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-11771.en.md b/changes/ce/fix-11771.en.md new file mode 100644 index 000000000..1df7503de --- /dev/null +++ b/changes/ce/fix-11771.en.md @@ -0,0 +1 @@ +Fixed validation of Bcrypt salt rounds in authentification management through the API/Dashboard. From 22f8df2eee775b61b93bcfbaa5e34d11f4ee03d8 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 17 Oct 2023 10:36:39 +0200 Subject: [PATCH 2/7] build: make use of rebar git cache in docker build --- .gitignore | 1 + build | 32 ++++++++++++++++++++++++++------ deploy/docker/Dockerfile | 32 ++++++++++++++++++++++---------- scripts/ensure-rebar3.sh | 4 +++- scripts/pre-compile.sh | 7 +++++-- scripts/update-bom.sh | 1 - 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index eecb62570..0e7c614a7 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ lux_logs/ bom.json ct_run*/ apps/emqx_conf/etc/emqx.conf.all.rendered* +rebar-git-cache.tar diff --git a/build b/build index 5c2bb556b..8b485f3b6 100755 --- a/build +++ b/build @@ -6,7 +6,11 @@ set -euo pipefail -[ "${DEBUG:-0}" -eq 1 ] && set -x +if [ "${DEBUG:-0}" -eq 1 ]; then + set -x + # set this for rebar3 + export DIAGNOSTIC=1 +fi PROFILE_ARG="$1" ARTIFACT="$2" @@ -449,17 +453,33 @@ make_docker() { if [ "${DOCKER_PUSH:-false}" = true ]; then DOCKER_BUILDX_ARGS+=(--push) fi + if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then + cache_tar="$(pwd)/rebar-git-cache.tar" + if [ ! -f "${cache_tar}" ]; then + pushd "${REBAR_GIT_CACHE_DIR}" >/dev/null + tar -cf "${cache_tar}" . + popd >/dev/null + fi + fi + if [ -n "${DEBUG:-}" ]; then + DOCKER_BUILDX_ARGS+=(--build-arg DEBUG="${DEBUG}" --progress=plain) + fi + # shellcheck disable=SC2015 [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true trap docker_cleanup EXIT { - echo '/_build' - echo '/deps' - echo '/*.lock' + echo '_build/' + echo 'deps/' + echo '*.lock' + echo '_packages/' + echo '.vs/' + echo '.vscode/' + echo 'lux_logs/' + echo '_upgrade_base/' } >> ./.dockerignore - set -x + echo "Docker build args: ${DOCKER_BUILDX_ARGS[*]}" docker buildx build "${DOCKER_BUILDX_ARGS[@]}" . - [[ "${DEBUG:-}" -eq 1 ]] || set +x echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag } diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index ec227cb61..b2dfbb1f6 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,23 +1,35 @@ ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11 ARG RUN_FROM=public.ecr.aws/debian/debian:11-slim FROM ${BUILD_FROM} AS builder +ARG DEBUG=0 COPY . /emqx ARG EMQX_NAME=emqx ARG PKG_VSN -ENV EMQX_RELUP=false -RUN export PROFILE=${EMQX_NAME%%-elixir} \ - && export EMQX_NAME1=$EMQX_NAME \ - && export EMQX_NAME=$PROFILE \ - && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ - && export EMQX_REL_FORM='docker' \ - && cd /emqx \ - && make $EMQX_NAME1 \ - && rm -f $EMQX_REL_PATH/*.tar.gz \ +ENV EMQX_RELUP=false +ENV DEBUG=${DEBUG} +ENV EMQX_REL_FORM='docker' + +WORKDIR /emqx/ + +RUN git config --global --add safe.directory '*' + +RUN if [ -f rebar-git-cache.tar ]; then \ + mkdir .cache && \ + tar -xf rebar-git-cache.tar -C .cache && \ + export REBAR_GIT_CACHE_DIR='/emqx/.cache' && \ + export REBAR_GIT_CACHE_REF_AUTOFILL=0 ;\ + fi \ + && export PROFILE=${EMQX_NAME%%-elixir} \ + && export EMQX_NAME1="${EMQX_NAME}" \ + && export EMQX_NAME=${PROFILE} \ + && export EMQX_REL_PATH="/emqx/_build/${EMQX_NAME}/rel/emqx" \ + && make ${EMQX_NAME1} \ + && rm -f ${EMQX_REL_PATH}/*.tar.gz \ && mkdir -p /emqx-rel \ - && mv $EMQX_REL_PATH /emqx-rel + && mv ${EMQX_REL_PATH} /emqx-rel FROM $RUN_FROM ARG EXTRA_DEPS='' diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 6eda16d30..12c492132 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,6 +2,8 @@ set -euo pipefail +[ "${DEBUG:-0}" -eq 1 ] && set -x + ## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image. ## we have to use an otp24-compiled rebar3 because the defination of record #application{} ## in systools.hrl is changed in otp24. @@ -14,7 +16,7 @@ case ${OTP_VSN} in VERSION="3.18.0-emqx-1" ;; 25*) - VERSION="3.19.0-emqx-8" + VERSION="3.19.0-emqx-9" ;; *) echo "Unsupporetd Erlang/OTP version $OTP_VSN" diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 1700dd8a4..632aabfe4 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -2,6 +2,8 @@ set -euo pipefail +[ "${DEBUG:-0}" -eq 1 ] && set -x + # NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise) # it might be with suffix such as -pkg etc. PROFILE_STR="${1}" @@ -28,5 +30,6 @@ curl -L --silent --show-error \ --output "apps/emqx_dashboard/priv/desc.zh.hocon" \ 'https://raw.githubusercontent.com/emqx/emqx-i18n/main/desc.zh.hocon' -# generate sbom -./scripts/update-bom.sh "$PROFILE_STR" ./rel +# TODO +# make sbom a build artifcat +# ./scripts/update-bom.sh "$PROFILE_STR" ./rel diff --git a/scripts/update-bom.sh b/scripts/update-bom.sh index 20ab45e22..d120b9bf4 100755 --- a/scripts/update-bom.sh +++ b/scripts/update-bom.sh @@ -8,4 +8,3 @@ PROFILE="$1" REL_DIR="$2" ./rebar3 as "$PROFILE" sbom -f -o "$REL_DIR/bom.json" - From e1c8317779a55c8a77f54a3f38ea6dca95bc2031 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 17 Oct 2023 17:38:02 +0300 Subject: [PATCH 3/7] chore(authn): remove dead code --- apps/emqx_auth_redis/src/emqx_authn_redis.erl | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 2f0948faf..960308ac9 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -18,100 +18,16 @@ -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("hocon/include/hoconsc.hrl"). --behaviour(hocon_schema). -behaviour(emqx_authn_provider). -export([ - namespace/0, - tags/0, - roots/0, - fields/1, - desc/1 -]). - --export([ - refs/0, - union_member_selector/1, create/2, update/2, authenticate/2, destroy/1 ]). -%%------------------------------------------------------------------------------ -%% Hocon Schema -%%------------------------------------------------------------------------------ - -namespace() -> "authn". - -tags() -> - [<<"Authentication">>]. - -%% used for config check when the schema module is resolved -roots() -> - [ - {?CONF_NS, - hoconsc:mk( - hoconsc:union(fun ?MODULE:union_member_selector/1), - #{} - )} - ]. - -fields(redis_single) -> - common_fields() ++ emqx_redis:fields(single); -fields(redis_cluster) -> - common_fields() ++ emqx_redis:fields(cluster); -fields(redis_sentinel) -> - common_fields() ++ emqx_redis:fields(sentinel). - -desc(redis_single) -> - ?DESC(single); -desc(redis_cluster) -> - ?DESC(cluster); -desc(redis_sentinel) -> - ?DESC(sentinel); -desc(_) -> - "". - -common_fields() -> - [ - {mechanism, emqx_authn_schema:mechanism(password_based)}, - {backend, emqx_authn_schema:backend(redis)}, - {cmd, fun cmd/1}, - {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1} - ] ++ emqx_authn_schema:common_fields(). - -cmd(type) -> string(); -cmd(desc) -> ?DESC(?FUNCTION_NAME); -cmd(required) -> true; -cmd(_) -> undefined. - -refs() -> - [ - hoconsc:ref(?MODULE, redis_single), - hoconsc:ref(?MODULE, redis_cluster), - hoconsc:ref(?MODULE, redis_sentinel) - ]. - -union_member_selector(all_union_members) -> - refs(); -union_member_selector({value, Value}) -> - refs(Value). - -refs(#{<<"redis_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, redis_single)]; -refs(#{<<"redis_type">> := <<"cluster">>}) -> - [hoconsc:ref(?MODULE, redis_cluster)]; -refs(#{<<"redis_type">> := <<"sentinel">>}) -> - [hoconsc:ref(?MODULE, redis_sentinel)]; -refs(_) -> - throw(#{ - field_name => redis_type, - expected => "single | cluster | sentinel" - }). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ From 2388d36b09dbec0c855abc3984100f28a4cc5397 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 18 Oct 2023 18:17:03 +0800 Subject: [PATCH 4/7] fix: allow viewers to change their own passwords --- .../src/emqx_dashboard_token.erl | 4 +-- .../src/emqx_dashboard_rbac.app.src | 2 +- .../src/emqx_dashboard_rbac.erl | 23 +++++++++------ .../test/emqx_dashboard_rbac_SUITE.erl | 28 +++++++++++++++++++ 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 1c840e90c..9a9875935 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -248,8 +248,8 @@ clean_expired_jwt(Now) -> -if(?EMQX_RELEASE_EDITION == ee). check_rbac(Req, JWT) -> - #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = _Username} = JWT, - case emqx_dashboard_rbac:check_rbac(Req, Extra) of + #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = Username} = JWT, + case emqx_dashboard_rbac:check_rbac(Req, Username, Extra) of true -> save_new_jwt(JWT); _ -> diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src index 190764e2f..ec8e6cd3f 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard_rbac, [ {description, "EMQX Dashboard RBAC"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index 28bd8960e..57132b65b 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -6,18 +6,18 @@ -include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). --export([check_rbac/2, role/1, valid_role/1]). +-export([check_rbac/3, role/1, valid_role/1]). -dialyzer({nowarn_function, role/1}). %%===================================================================== %% API -check_rbac(Req, Extra) -> +check_rbac(Req, Username, Extra) -> Role = role(Extra), Method = cowboy_req:method(Req), AbsPath = cowboy_req:path(Req), case emqx_dashboard_swagger:get_relative_uri(AbsPath) of {ok, Path} -> - check_rbac(Role, Method, Path); + check_rbac(Role, Method, Path, Username); _ -> false end. @@ -41,14 +41,21 @@ valid_role(Role) -> {error, <<"Role does not exist">>} end. %% =================================================================== -check_rbac(?ROLE_SUPERUSER, _, _) -> +check_rbac(?ROLE_SUPERUSER, _, _, _) -> true; -check_rbac(?ROLE_VIEWER, <<"GET">>, _) -> +check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) -> true; -%% this API is a special case -check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) -> +%% everyone should allow to logout +check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>, _) -> true; -check_rbac(_, _, _) -> +%% viewer should allow to change self password, +%% superuser should allow to change any user +check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/users/", SubPath/binary>>, Username) -> + case binary:split(SubPath, <<"/">>, [global]) of + [Username, <<"change_pwd">>] -> true; + _ -> false + end; +check_rbac(_, _, _, _) -> false. role_list() -> diff --git a/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl b/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl index b1a51a3c9..eeac8dadf 100644 --- a/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl +++ b/apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl @@ -160,6 +160,34 @@ t_login_out(_) -> {ok, Username} = emqx_dashboard_admin:verify_token(FakeReq, Token), ok. +t_change_pwd(_) -> + Viewer1 = <<"viewer1">>, + Viewer2 = <<"viewer2">>, + SuperUser = <<"super_user">>, + Password = <<"public_www1">>, + Desc = <<"desc">>, + {ok, _} = emqx_dashboard_admin:add_user(Viewer1, Password, ?ROLE_VIEWER, Desc), + {ok, _} = emqx_dashboard_admin:add_user(Viewer2, Password, ?ROLE_VIEWER, Desc), + {ok, _} = emqx_dashboard_admin:add_user(SuperUser, Password, ?ROLE_SUPERUSER, Desc), + {ok, ?ROLE_VIEWER, Viewer1Token} = emqx_dashboard_admin:sign_token(Viewer1, Password), + {ok, ?ROLE_SUPERUSER, SuperToken} = emqx_dashboard_admin:sign_token(SuperUser, Password), + %% viewer can change own password + ?assertEqual({ok, Viewer1}, change_pwd(Viewer1Token, Viewer1)), + %% viewer can't change other's password + ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, Viewer2)), + ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, SuperUser)), + %% superuser can change other's password + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer1)), + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer2)), + ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, SuperUser)), + ok. + +change_pwd(Token, Username) -> + Path = "/users/" ++ binary_to_list(Username) ++ "/change_pwd", + Path1 = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri(Path)), + Req = #{method => <<"POST">>, path => Path1}, + emqx_dashboard_admin:verify_token(Req, Token). + add_default_superuser() -> {ok, _NewUser} = emqx_dashboard_admin:add_user( ?DEFAULT_SUPERUSER, From 81e10c6748a6efbbe42d01dff1c974bce2edbe94 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 18 Oct 2023 18:43:11 +0800 Subject: [PATCH 5/7] chore: add changelog for 11785 --- changes/ce/feat-11785.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-11785.en.md diff --git a/changes/ce/feat-11785.en.md b/changes/ce/feat-11785.en.md new file mode 100644 index 000000000..765ce6ea0 --- /dev/null +++ b/changes/ce/feat-11785.en.md @@ -0,0 +1 @@ +Allow viewer to change their own passwords, viewer can't change other's password. From ad09ca9d6d1e47c0b17c7ac651a4cc78c073788b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 18 Oct 2023 13:26:53 +0200 Subject: [PATCH 6/7] refactor(nodetool): only add libs when necessary --- bin/nodetool | 3 ++- changes/ce/feat-11787.en.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changes/ce/feat-11787.en.md diff --git a/bin/nodetool b/bin/nodetool index ab2210aa5..c0d5b0025 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -21,12 +21,13 @@ main(Args) -> ok end end, - ok = add_libs_dir(), case Args of ["hocon" | Rest] -> + ok = add_libs_dir(), %% forward the call to hocon_cli hocon_cli:main(Rest); ["check_license_key", Key0] -> + ok = add_libs_dir(), Key = cleanup_key(Key0), check_license(#{key => Key}); _ -> diff --git a/changes/ce/feat-11787.en.md b/changes/ce/feat-11787.en.md new file mode 100644 index 000000000..2dc3efc73 --- /dev/null +++ b/changes/ce/feat-11787.en.md @@ -0,0 +1,3 @@ +Improve `emqx` command performance. + +Avoid loading EMQX application code in `nodetool` script unless necessary. From 5b9866f63096ce7292c271cdafbdf2e0732a8ddf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 18 Oct 2023 16:08:29 -0300 Subject: [PATCH 7/7] fix(coap): increase received packet counter for keepalive Fixes https://emqx.atlassian.net/browse/EMQX-11193 Fixes https://github.com/emqx/emqx/issues/11779 --- .../src/emqx_coap_channel.erl | 18 +++-- .../test/emqx_coap_SUITE.erl | 73 +++++++++++++++++-- changes/ce/fix-11791.en.md | 1 + 3 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 changes/ce/fix-11791.en.md diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 467ac20a2..5e3461c52 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -86,7 +86,6 @@ -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). -define(DEF_IDLE_TIME, timer:seconds(30)). --define(GET_IDLE_TIME(Cfg), maps:get(idle_timeout, Cfg, ?DEF_IDLE_TIME)). -import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). @@ -150,8 +149,7 @@ init( mountpoint => Mountpoint } ), - %% FIXME: it should coap.hearbeat instead of idle_timeout? - Heartbeat = ?GET_IDLE_TIME(Config), + Heartbeat = maps:get(heartbeat, Config, ?DEF_IDLE_TIME), #channel{ ctx = Ctx, conninfo = ConnInfo, @@ -179,8 +177,8 @@ send_request(Channel, Request) -> | {ok, replies(), channel()} | {shutdown, Reason :: term(), channel()} | {shutdown, Reason :: term(), replies(), channel()}. -handle_in(Msg, ChannleT) -> - Channel = ensure_keepalive_timer(ChannleT), +handle_in(Msg, Channel0) -> + Channel = ensure_keepalive_timer(Channel0), case emqx_coap_message:is_request(Msg) of true -> check_auth_state(Msg, Channel); @@ -321,6 +319,9 @@ handle_call(Req, _From, Channel) -> handle_cast(close, Channel) -> ?SLOG(info, #{msg => "close_connection"}), shutdown(normal, Channel); +handle_cast(inc_recv_pkt, Channel) -> + _ = emqx_pd:inc_counter(recv_pkt, 1), + {ok, Channel}; handle_cast(Req, Channel) -> ?SLOG(error, #{msg => "unexpected_cast", cast => Req}), {ok, Channel}. @@ -455,6 +456,13 @@ check_token( Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), {shutdown, normal, Reply, Channel}; true -> + %% hack: since each message request can spawn a new connection + %% process, we can't rely on the `inc_incoming_stats' call in + %% `emqx_gateway_conn:handle_incoming' to properly keep track of + %% bumping incoming requests for an existing channel. Since this + %% number is used by keepalive, we have to bump it inside the + %% requested channel/connection pid so heartbeats actually work. + emqx_gateway_cm:cast(coap, ReqClientId, inc_recv_pkt), call_session(handle_request, Msg, Channel) end; _ -> diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 4459d84f1..c066b84ff 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -83,10 +83,26 @@ init_per_testcase(t_connection_with_authn_failed, Config) -> fun(_) -> {error, bad_username_or_password} end ), Config; +init_per_testcase(t_heartbeat, Config) -> + NewHeartbeat = 800, + OldConf = emqx:get_raw_config([gateway, coap]), + {ok, _} = emqx_gateway_conf:update_gateway( + coap, + OldConf#{<<"heartbeat">> => <<"800ms">>} + ), + [ + {old_conf, OldConf}, + {new_heartbeat, NewHeartbeat} + | Config + ]; init_per_testcase(_, Config) -> ok = meck:new(emqx_access_control, [passthrough]), Config. +end_per_testcase(t_heartbeat, Config) -> + OldConf = ?config(old_conf, Config), + {ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf), + ok; end_per_testcase(_, Config) -> ok = meck:unload(emqx_access_control), Config. @@ -123,13 +139,49 @@ t_connection(_) -> ), %% heartbeat - HeartURI = - ?MQTT_PREFIX ++ - "/connection?clientid=client1&token=" ++ - Token, + {ok, changed, _} = send_heartbeat(Token), - ?LOGT("send heartbeat request:~ts~n", [HeartURI]), - {ok, changed, _} = er_coap_client:request(put, HeartURI), + disconnection(Channel, Token), + + timer:sleep(100), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ) + end, + do(Action). + +t_heartbeat(Config) -> + Heartbeat = ?config(new_heartbeat, Config), + Action = fun(Channel) -> + Token = connection(Channel), + + timer:sleep(100), + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + %% must keep client connection alive + Delay = Heartbeat div 2, + lists:foreach( + fun(_) -> + ?assertMatch({ok, changed, _}, send_heartbeat(Token)), + timer:sleep(Delay) + end, + lists:seq(1, 5) + ), + + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + timer:sleep(Heartbeat * 2), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), disconnection(Channel, Token), @@ -491,6 +543,15 @@ t_connectionless_pubsub(_) -> %%-------------------------------------------------------------------- %% helpers +send_heartbeat(Token) -> + HeartURI = + ?MQTT_PREFIX ++ + "/connection?clientid=client1&token=" ++ + Token, + + ?LOGT("send heartbeat request:~ts~n", [HeartURI]), + er_coap_client:request(put, HeartURI). + connection(Channel) -> URI = ?MQTT_PREFIX ++ diff --git a/changes/ce/fix-11791.en.md b/changes/ce/fix-11791.en.md new file mode 100644 index 000000000..983347605 --- /dev/null +++ b/changes/ce/fix-11791.en.md @@ -0,0 +1 @@ +Fixed an issue that prevented heartbeats from correctly keeping the CoAP Gateway connections alive.