diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index a0f85fa22..c4b752da1 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -175,26 +175,6 @@ EOF cat /var/log/emqx/emqx.log.1 || true exit 1 fi - - if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \ - || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then - if ! service emqx start; then - cat /var/log/emqx/erlang.log.1 || true - cat /var/log/emqx/emqx.log.1 || true - exit 1 - fi - IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/api/v5/status >/dev/null 2>&1; do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx service error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done - service emqx stop - fi } relup_test(){ diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 8c9d056cc..ae3d12c64 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -1,6 +1,6 @@ MYSQL_TAG=8 REDIS_TAG=6 -MONGO_TAG=4 +MONGO_TAG=5 PGSQL_TAG=13 LDAP_TAG=2.4.50 diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local index d5ef99d66..096da64c5 100644 --- a/.ci/docker-compose-file/Makefile.local +++ b/.ci/docker-compose-file/Makefile.local @@ -14,7 +14,7 @@ up: env \ MYSQL_TAG=8 \ REDIS_TAG=6 \ - MONGO_TAG=4 \ + MONGO_TAG=5 \ PGSQL_TAG=13 \ LDAP_TAG=2.4.50 \ docker-compose \ diff --git a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml index 494b42ce4..5bba6147c 100644 --- a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml @@ -2,11 +2,9 @@ version: '3.9' services: mongo_server: - container_name: mongo + container_name: mongo image: mongo:${MONGO_TAG} restart: always - environment: - MONGO_INITDB_DATABASE: mqtt networks: - emqx_bridge ports: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 2f0137428..f1cce9364 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang23: container_name: erlang23 - image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-3:23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: @@ -23,7 +23,7 @@ services: erlang24: container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04 env_file: - conf.env environment: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index bf972f920..77376f02c 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -19,7 +19,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" outputs: old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }} @@ -129,7 +129,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 macos: - macos-11 - macos-10.15 @@ -215,7 +215,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions + - 24.1.5-3 # we test with OTP 23, but only build package on OTP 24 versions arch: - amd64 - arm64 @@ -301,7 +301,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/5.0-3:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -336,7 +336,7 @@ jobs: - emqx-enterprise # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.1.5-2 # update to latest + - 24.1.5-3 # update to latest steps: - uses: actions/download-artifact@v2 @@ -377,7 +377,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile @@ -405,7 +405,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index da62e296b..aaa56b30b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -24,12 +24,12 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 os: - ubuntu20.04 - centos7 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v1 @@ -55,7 +55,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.1.5-2 + - 24.1.5-3 macos: - macos-11 - macos-10.15 diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index c1a457665..2318b9938 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index caa4e14ad..1aba6c594 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -13,10 +13,10 @@ jobs: matrix: otp: - 23.3.4.9-3 - - 24.1.5-2 + - 24.1.5-3 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp }}-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 45f8d8965..2ca594de6 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -14,7 +14,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + container: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 steps: - uses: actions/checkout@v2 @@ -55,7 +55,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 run: | make ${{ matrix.profile }}-docker - name: run emqx @@ -100,7 +100,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-alpine3.14 run: | make ${{ matrix.profile }}-docker echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 18d995d44..3b796d476 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -19,10 +19,10 @@ jobs: - emqx - emqx-enterprise otp_vsn: - - 24.1.5-2 + - 24.1.5-3 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:${{ matrix.otp_vsn }}-ubuntu20.04" defaults: run: diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d1f8bf577..bb0fb1c82 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -16,7 +16,7 @@ jobs: strategy: matrix: emqx_builder: - - 5.0-2:24.1.5-2 # run dialyzer on latest OTP + - 5.0-3:24.1.5-3 # run dialyzer on latest OTP runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" @@ -32,7 +32,7 @@ jobs: strategy: matrix: emqx_builder: - - 5.0-2:24.1.5-2 + - 5.0-3:24.1.5-3 runs-on: ubuntu-20.04 container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" @@ -55,12 +55,14 @@ jobs: - uses: actions/checkout@v2 - name: docker compose up env: + MONGO_TAG: 5 MYSQL_TAG: 8 PGSQL_TAG: 13 REDIS_TAG: 6 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | docker-compose \ + -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ diff --git a/.tool-versions b/.tool-versions index 6d9b11e28..a6568713b 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -erlang 24.1.5-2 +erlang 24.1.5-3 diff --git a/Makefile b/Makefile index e381db3ed..52e4f3ffc 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) ## app/name-ct targets are intended for local tests hence cover is not enabled .PHONY: $(APPS:%=%-ct) define gen-app-ct-target -$1-ct: +$1-ct: conf-segs $(REBAR) ct --name $(CT_NODE_NAME) -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) diff --git a/apps/emqx/include/emqx_authentication.hrl b/apps/emqx/include/emqx_authentication.hrl new file mode 100644 index 000000000..948842433 --- /dev/null +++ b/apps/emqx/include/emqx_authentication.hrl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_AUTHENTICATION_HRL). +-define(EMQX_AUTHENTICATION_HRL, true). + +%% config root name all auth providers have to agree on. +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, "authentication"). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication). +-define(EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, <<"authentication">>). + +%% key to a persistent term which stores a module name in order to inject +%% schema module at run-time to keep emqx app's compile time purity. +%% see emqx_schema.erl for more details +%% and emqx_conf_schema for an examples +-define(EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, emqx_authentication_schema_module). + +-endif. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 59c5cf045..f8c6c23ce 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -17,10 +17,10 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.20.6"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.22.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} - , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} + , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.16.0"}}} ]}. {plugins, [{rebar3_proper, "0.12.1"}]}. diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 914651535..35b3767e0 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -63,4 +63,5 @@ do_authorize(ClientInfo, PubSub, Topic) -> -compile({inline, [run_hooks/3]}). run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). + ok = emqx_metrics:inc(Name), + emqx_hooks:run_fold(Name, Args, Acc). diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl index 77a5e2cee..1607497da 100644 --- a/apps/emqx/src/emqx_authentication.erl +++ b/apps/emqx/src/emqx_authentication.erl @@ -24,9 +24,12 @@ -include("emqx.hrl"). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + %% The authentication entrypoint. -export([ authenticate/2 ]). @@ -383,8 +386,8 @@ list_users(ChainName, AuthenticatorID, Params) -> %%-------------------------------------------------------------------- init(_Opts) -> - ok = emqx_config_handler:add_handler([authentication], ?MODULE), - ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + ok = emqx_config_handler:add_handler([?CONF_ROOT], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', ?CONF_ROOT], ?MODULE), {ok, #{hooked => false, providers => #{}}}. handle_call(get_providers, _From, #{providers := Providers} = State) -> @@ -496,8 +499,8 @@ terminate(Reason, _State) -> Other -> ?SLOG(error, #{msg => "emqx_authentication_terminating", reason => Other}) end, - emqx_config_handler:remove_handler([authentication]), - emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + emqx_config_handler:remove_handler([?CONF_ROOT]), + emqx_config_handler:remove_handler([listeners, '?', '?', ?CONF_ROOT]), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index a7fa5673a..795dd060e 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -34,6 +34,7 @@ -export_type([config/0]). -include("logger.hrl"). +-include("emqx_authentication.hrl"). -type parsed_config() :: #{mechanism := atom(), backend => atom(), @@ -132,9 +133,9 @@ do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position} check_configs(Configs) -> Providers = emqx_authentication:get_providers(), - lists:map(fun(C) -> do_check_conifg(C, Providers) end, Configs). + lists:map(fun(C) -> do_check_config(C, Providers) end, Configs). -do_check_conifg(Config, Providers) -> +do_check_config(Config, Providers) -> Type = authn_type(Config), case maps:get(Type, Providers, false) of false -> @@ -143,19 +144,20 @@ do_check_conifg(Config, Providers) -> providers => Providers}), throw({unknown_authn_type, Type}); Module -> - do_check_conifg(Type, Config, Module) + do_check_config(Type, Config, Module) end. -do_check_conifg(Type, Config, Module) -> +do_check_config(Type, Config, Module) -> F = case erlang:function_exported(Module, check_config, 1) of true -> fun Module:check_config/1; false -> fun(C) -> - #{config := R} = - hocon_schema:check_plain(Module, #{<<"config">> => C}, + Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME), + R = hocon_schema:check_plain(Module, #{Key => C}, #{atom_key => true}), - R + maps:get(AtomKey, R) end end, try @@ -261,8 +263,8 @@ authn_type(#{mechanism := M}) -> atom(M); authn_type(#{<<"mechanism">> := M, <<"backend">> := B}) -> {atom(M), atom(B)}; authn_type(#{<<"mechanism">> := M}) -> atom(M). -atom(Bin) -> - binary_to_existing_atom(Bin, utf8). +atom(A) when is_atom(A) -> A; +atom(Bin) -> binary_to_existing_atom(Bin, utf8). %% The relative dir for ssl files. certs_dir(ChainName, ConfigOrID) -> diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 8806d4bc3..9979629bf 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -268,23 +268,39 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> +init_load(SchemaMod, RawConf) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - %% check and save configs - {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf0), + %% check configs agains the schema, with environment variables applied on top + {_AppEnvs, CheckedConf} = + check_config(SchemaMod, RawConf, #{apply_override_envs => true}), + %% fill default values for raw config + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RootNames = get_root_names(), ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), - maps:with(get_root_names(), RawConf0)). + maps:with(RootNames, RawConfWithEnvs)). include_dirs() -> [filename:join(emqx:data_dir(), "configs")]. +merge_envs(SchemaMod, RawConf) -> + Opts = #{logger => fun(_, _) -> ok end, %% everything should have been logged already when check_config + nullable => true, %% TODO: evil, remove, nullable should be declared in schema + format => map, + apply_override_envs => true + }, + hocon_schema:merge_env_overrides(SchemaMod, RawConf, all, Opts). + -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} when AppEnvs :: app_envs(), CheckedConf :: config(). check_config(SchemaMod, RawConf) -> - Opts = #{return_plain => true, - nullable => true, - format => map - }, + check_config(SchemaMod, RawConf, #{}). + +check_config(SchemaMod, RawConf, Opts0) -> + Opts1 = #{return_plain => true, + nullable => true, %% TODO: evil, remove, nullable should be declared in schema + format => map + }, + Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawConf, Opts), {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. @@ -312,13 +328,15 @@ read_override_conf(#{} = Opts) -> File = override_conf_file(Opts), load_hocon_file(File, map). -override_conf_file(Opts) -> +override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, local) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, - application:get_env(emqx, Key, undefined). + application:get_env(emqx, Key, undefined); +override_conf_file(Which) when is_atom(Which) -> + application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. save_schema_mod_and_names(SchemaMod) -> diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 2fe1b6d1a..647d076d1 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -248,8 +248,8 @@ parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) -> }, {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4), {Username, Rest6} = parse_utf8_string(Rest5, bool(UsernameFlag)), - {Passsword, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)), - ConnPacket1#mqtt_packet_connect{username = Username, password = Passsword}; + {Password, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)), + ConnPacket1#mqtt_packet_connect{username = Username, password = Password}; parse_packet(#mqtt_packet_header{type = ?CONNACK}, <>, #{version := Ver}) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2b9a76fe3..2869dfc75 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -111,7 +111,7 @@ current_conns(ID, ListenOn) -> {Type, Name} = parse_listener_id(ID), current_conns(Type, Name, ListenOn). -current_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> +current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_current_connections({listener_id(Type, Name), ListenOn}); current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); @@ -122,7 +122,7 @@ max_conns(ID, ListenOn) -> {Type, Name} = parse_listener_id(ID), max_conns(Type, Name, ListenOn). -max_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> +max_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_max_connections({listener_id(Type, Name), ListenOn}); max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(max_connections, ranch:info(listener_id(Type, Name))); diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index e0cfac7af..263035db5 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -96,8 +96,8 @@ handle_info({timeout, _Timer, check}, State) -> _ = case emqx_vm:cpu_util() of %% TODO: should be improved? 0 -> ok; Busy when Busy >= CPUHighWatermark -> - Usage = io_lib:format("~p%", [Busy]), - Message = [Usage, " cpu usage"], + Usage = list_to_binary(io_lib:format("~.2f%", [Busy])), + Message = <>, emqx_alarm:activate(high_cpu_usage, #{ usage => Usage, @@ -107,8 +107,8 @@ handle_info({timeout, _Timer, check}, State) -> Message), start_check_timer(); Busy when Busy =< CPULowWatermark -> - Usage = io_lib:format("~p%", [Busy]), - Message = [Usage, " cpu usage"], + Usage = list_to_binary(io_lib:format("~.2f%", [Busy])), + Message = <>, emqx_alarm:deactivate(high_cpu_usage, #{ usage => Usage, diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e470ff478..1c471da90 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -22,6 +22,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include("emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type duration() :: integer(). @@ -105,11 +106,29 @@ and can not be deleted.""" The configs here work as default values which can be overriden in zone configs""" })} - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication( -"""Default authentication configs for all MQTT listeners.
+"""Default authentication configs for all MQTT listeners. +
For per-listener overrides see authentication -in listener configs""")} +in listener configs +
+
+EMQ X can be configured with: +
+
    +
  • []: The default value, it allows *ALL* logins
    • +
  • one: For example {enable:true,backend:\"built-in-database\",mechanism=\"password-based\"}
    • +
  • chain: An array of structs.
    • +
+
+When a chain is configured, the login credentials are checked against the backends +per the configured order, until an 'allow' or 'deny' decision can be made. +
+If there is no decision after a full chain exhaustion, the login is rejected. +""")} + %% NOTE: authorization schema here is only to keep emqx app prue + %% the full schema for EMQ X node is injected in emqx_conf_schema. , {"authorization", sc(ref("authorization"), #{})} @@ -972,7 +991,7 @@ mqtt_listener() -> sc(duration(), #{}) } - , {"authentication", + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication("Per-listener authentication override") } ]. @@ -1231,16 +1250,18 @@ ciphers_schema(Default) -> false -> fun validate_ciphers/1 end , desc => -"""TLS cipher suite names separated by comma, or as an array of strings +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. \"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256\" or -[\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"][\"TLS_AES_256_GCM_SHA384\",\"TLS_AES_128_GCM_SHA256\"].
Ciphers (and their ordering) define the way in which the -client and server encrypts information over the wire. +client and server encrypts information over the network connection. Selecting a good cipher suite is critical for the application's data security, confidentiality and performance. -The names should be in OpenSSL sting format (not RFC format). -Default values and examples proveded by EMQ X config + +The names should be in OpenSSL string format (not RFC format). +All default values and examples proveded by EMQ X config documentation are all in OpenSSL format.
NOTE: Certain cipher suites are only compatible with @@ -1436,12 +1457,23 @@ str(S) when is_list(S) -> S. authentication(Desc) -> - #{ type => hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])) - , desc => iolist_to_binary([Desc, "
", """ + %% authentication schemais lazy to make it more 'plugable' + %% the type checks are done in emqx_auth application when it boots. + %% and in emqx_authentication_config module for rutime changes. + Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])), + %% as the type is lazy, the runtime module injection from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY + %% is for now only affecting document generation. + %% maybe in the future, we can find a more straightforward way to support + %% * document generation (at compile time) + %% * type checks before boot (in bin/emqx config generation) + %% * type checks at runtime (when changing configs via management API) + #{ type => case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of + undefined -> Default; + Module -> hoconsc:lazy(Module:root_type()) + end + , desc => iolist_to_binary([Desc, """ Authentication can be one single authenticator instance or a chain of authenticators as an array. When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.
-EMQ X comes with a set of pre-built autenticators, for more details, see -autenticator_config """]) }. diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 00d26d147..a58c179b7 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -54,11 +54,11 @@ wildcard(Topic) when is_binary(Topic) -> wildcard(words(Topic)); wildcard([]) -> false; -wildcard(['#'|_]) -> +wildcard(['#' | _]) -> true; -wildcard(['+'|_]) -> +wildcard(['+' | _]) -> true; -wildcard([_H|T]) -> +wildcard([_H | T]) -> wildcard(T). %% @doc Match Topic name with filter. @@ -73,17 +73,17 @@ match(Name, Filter) when is_binary(Name), is_binary(Filter) -> match(words(Name), words(Filter)); match([], []) -> true; -match([H|T1], [H|T2]) -> +match([H | T1], [H | T2]) -> match(T1, T2); -match([_H|T1], ['+'|T2]) -> +match([_H | T1], ['+' | T2]) -> match(T1, T2); match(_, ['#']) -> true; -match([_H1|_], [_H2|_]) -> +match([_H1 | _], [_H2 | _]) -> false; -match([_H1|_], []) -> +match([_H1 | _], []) -> false; -match([], [_H|_T2]) -> +match([], [_H | _T2]) -> false. %% @doc Validate topic name or filter @@ -110,13 +110,13 @@ validate2([]) -> true; validate2(['#']) -> % end with '#' true; -validate2(['#'|Words]) when length(Words) > 0 -> +validate2(['#' | Words]) when length(Words) > 0 -> error('topic_invalid_#'); -validate2([''|Words]) -> +validate2(['' | Words]) -> validate2(Words); -validate2(['+'|Words]) -> +validate2(['+' | Words]) -> validate2(Words); -validate2([W|Words]) -> +validate2([W | Words]) -> validate3(W) andalso validate2(Words). validate3(<<>>) -> @@ -164,7 +164,7 @@ word(<<"#">>) -> '#'; word(Bin) -> Bin. %% @doc '$SYS' Topic. --spec(systop(atom()|string()|binary()) -> topic()). +-spec(systop(atom() | string() | binary()) -> topic()). systop(Name) when is_atom(Name); is_list(Name) -> iolist_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name])); systop(Name) when is_binary(Name) -> @@ -175,10 +175,10 @@ feed_var(Var, Val, Topic) -> feed_var(Var, Val, words(Topic), []). feed_var(_Var, _Val, [], Acc) -> join(lists:reverse(Acc)); -feed_var(Var, Val, [Var|Words], Acc) -> - feed_var(Var, Val, Words, [Val|Acc]); -feed_var(Var, Val, [W|Words], Acc) -> - feed_var(Var, Val, Words, [W|Acc]). +feed_var(Var, Val, [Var | Words], Acc) -> + feed_var(Var, Val, Words, [Val | Acc]); +feed_var(Var, Val, [W | Words], Acc) -> + feed_var(Var, Val, Words, [W | Acc]). -spec(join(list(binary())) -> binary()). join([]) -> @@ -218,4 +218,3 @@ parse(TopicFilter = <<"$share/", Rest/binary>>, Options) -> end; parse(TopicFilter, Options) -> {TopicFilter, Options}. - diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index 06e17513b..91b3e54f9 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -57,6 +57,7 @@ sl_alloc, ll_alloc, fix_alloc, + literal_alloc, std_alloc ]). diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 10a4e4091..2c28a6076 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -25,18 +25,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("typerefl/include/types.hrl"). - --export([ roots/0, fields/1 ]). - --export([ create/2 - , update/2 - , authenticate/2 - , destroy/1 - , check_config/1 - ]). +-include("emqx_authentication.hrl"). -define(AUTHN, emqx_authentication). -define(config(KEY), (fun() -> {KEY, _V_} = lists:keyfind(KEY, 1, Config), _V_ end)()). +-define(CONF_ROOT, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %%------------------------------------------------------------------------------ %% Hocon Schema @@ -250,7 +243,7 @@ t_update_config({init, Config}) -> {"auth2", AuthNType2} | Config]; t_update_config(Config) when is_list(Config) -> - emqx_config_handler:add_handler([authentication], emqx_authentication), + emqx_config_handler:add_handler([?CONF_ROOT], emqx_authentication), ok = register_provider(?config("auth1"), ?MODULE), ok = register_provider(?config("auth2"), ?MODULE), Global = ?config(global), @@ -267,7 +260,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig1})), ?assertMatch( {ok, #{id := ID1, state := #{mark := 1}}}, @@ -275,7 +268,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + update_config([?CONF_ROOT], {create_authenticator, Global, AuthenticatorConfig2})), ?assertMatch( {ok, #{id := ID2, state := #{mark := 1}}}, @@ -283,7 +276,7 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], + update_config([?CONF_ROOT], {update_authenticator, Global, ID1, @@ -296,25 +289,25 @@ t_update_config(Config) when is_list(Config) -> ?assertMatch( {ok, _}, - update_config([authentication], {move_authenticator, Global, ID2, top})), + update_config([?CONF_ROOT], {move_authenticator, Global, ID2, top})), ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), - ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertMatch({ok, _}, update_config([?CONF_ROOT], {delete_authenticator, Global, ID1})), ?assertEqual( {error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), ?assertMatch( {ok, _}, - update_config([authentication], {delete_authenticator, Global, ID2})), + update_config([?CONF_ROOT], {delete_authenticator, Global, ID2})), ?assertEqual( {error, {not_found, {authenticator, ID2}}}, ?AUTHN:lookup_authenticator(Global, ID2)), ListenerID = 'tcp:default', - ConfKeyPath = [listeners, tcp, default, authentication], + ConfKeyPath = [listeners, tcp, default, ?CONF_ROOT], ?assertMatch( {ok, _}, diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 284cea784..5a626ed36 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -132,7 +132,7 @@ basic_conf() -> zones => zone_conf() }. -set_test_listenser_confs() -> +set_test_listener_confs() -> Conf = emqx_config:get([]), emqx_config:put(basic_conf()), Conf. @@ -179,7 +179,7 @@ end_per_suite(_Config) -> ]). init_per_testcase(_TestCase, Config) -> - NewConf = set_test_listenser_confs(), + NewConf = set_test_listener_confs(), [{config, NewConf}|Config]. end_per_testcase(_TestCase, Config) -> diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index d006a1a10..90f1bdca1 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -59,7 +59,7 @@ init_per_suite(Config) -> ok = meck:expect(emqx_alarm, deactivate, fun(_) -> ok end), ok = meck:expect(emqx_alarm, deactivate, fun(_, _) -> ok end), - emqx_channel_SUITE:set_test_listenser_confs(), + emqx_channel_SUITE:set_test_listener_confs(), Config. end_per_suite(_Config) -> diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 649fc0065..e4e04fb6a 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,10 +38,37 @@ end_per_suite(_Config) -> application:stop(esockd), application:stop(cowboy). +init_per_testcase(Case, Config) + when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp -> + {ok, _} = emqx_config_handler:start_link(), + PrevListeners = emqx_config:get([listeners, tcp], #{}), + PrevRateLimit = emqx_config:get([rate_limit], #{}), + emqx_config:put([listeners, tcp], #{ listener_test => + #{ bind => {"127.0.0.1", 9999} + , max_connections => 4321 + } + }), + emqx_config:put([rate_limit], #{max_conn_rate => 1000}), + ListenerConf = #{ bind => {"127.0.0.1", 9999} + }, + ok = emqx_listeners:start(), + [ {listener_conf, ListenerConf} + , {prev_listener_conf, PrevListeners} + , {prev_rate_limit_conf, PrevRateLimit} + | Config]; init_per_testcase(_, Config) -> {ok, _} = emqx_config_handler:start_link(), Config. +end_per_testcase(Case, Config) + when Case =:= t_max_conns_tcp; Case =:= t_current_conns_tcp -> + PrevListener = ?config(prev_listener_conf, Config), + PrevRateLimit = ?config(prev_rate_limit_conf, Config), + emqx_config:put([listeners, tcp], PrevListener), + emqx_config:put([rate_limit], PrevRateLimit), + emqx_listeners:stop(), + _ = emqx_config_handler:stop(), + ok; end_per_testcase(_, _Config) -> _ = emqx_config_handler:stop(), ok. @@ -56,6 +84,14 @@ t_restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +t_max_conns_tcp(_) -> + %% Note: Using a string representation for the bind address like + %% "127.0.0.1" does not work + ?assertEqual(4321, emqx_listeners:max_conns('tcp:listener_test', {{127,0,0,1}, 9999})). + +t_current_conns_tcp(_) -> + ?assertEqual(0, emqx_listeners:current_conns('tcp:listener_test', {{127,0,0,1}, 9999})). + render_config_file() -> Path = local_path(["etc", "emqx.conf"]), {ok, Temp} = file:read_file(Path), @@ -101,4 +137,3 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index fd9640aeb..0fa73ebe2 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -245,7 +245,7 @@ receive_messages(Count, Msgs) -> receive_messages(Count-1, [Msg|Msgs]); _Other -> receive_messages(Count, Msgs) - after 1000 -> + after 5000 -> Msgs end. @@ -576,7 +576,7 @@ t_publish_while_client_is_gone(Config) -> | Config]), {ok, _} = emqtt:ConnFun(Client2), Msgs = receive_messages(2), - ?assertEqual(length(Msgs), 2), + ?assertMatch([_, _], Msgs), [Msg2, Msg1] = Msgs, ?assertEqual({ok, iolist_to_binary(Payload1)}, maps:find(payload, Msg1)), ?assertEqual({ok, 2}, maps:find(qos, Msg1)), @@ -768,7 +768,7 @@ t_lost_messages_because_of_gc(Config) -> check_snabbkaffe_vanilla(Trace) -> ResumeTrace = [T || #{?snk_kind := K} = T <- Trace, - re:run(atom_to_list(K), "^ps_") /= nomatch], + re:run(to_list(K), "^ps_") /= nomatch], ?assertMatch([_|_], ResumeTrace), [_Sid] = lists:usort(?projection(sid, ResumeTrace)), %% Check internal flow of the emqx_cm resuming @@ -811,6 +811,10 @@ check_snabbkaffe_vanilla(Trace) -> [Markers] = ?projection(markers, ?of_kind(ps_node_markers, Trace)), ?assertMatch([_], Markers). +to_list(L) when is_list(L) -> L; +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(B) when is_binary(B) -> binary_to_list(B). + %%-------------------------------------------------------------------- %% Snabbkaffe tests %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl index 70ae5310e..8e29b8201 100644 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ b/apps/emqx/test/emqx_session_SUITE.erl @@ -29,7 +29,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Config) -> - emqx_channel_SUITE:set_test_listenser_confs(), + emqx_channel_SUITE:set_test_listener_confs(), ok = meck:new([emqx_hooks, emqx_metrics, emqx_broker], [passthrough, no_history, no_link]), ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index 76b95d9bc..49108f83e 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -20,6 +20,7 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_placeholder.hrl"). -import(emqx_topic, [ wildcard/1 @@ -183,9 +184,11 @@ t_feed_var(_) -> ?assertEqual(<<"$queue/client/clientId">>, feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)), ?assertEqual(<<"username/test/client/x">>, - feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>)), + feed_var( ?PH_USERNAME, <<"test">> + , <<"username/", ?PH_USERNAME/binary, "/client/x">>)), ?assertEqual(<<"username/test/client/clientId">>, - feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>)). + feed_var( ?PH_CLIENTID, <<"clientId">> + , <<"username/test/client/", ?PH_CLIENTID/binary>>)). long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 66666)]). diff --git a/apps/emqx/test/emqx_trace_handler_SUITE.erl b/apps/emqx/test/emqx_trace_handler_SUITE.erl index e010b6c5e..950f0faf6 100644 --- a/apps/emqx/test/emqx_trace_handler_SUITE.erl +++ b/apps/emqx/test/emqx_trace_handler_SUITE.erl @@ -199,6 +199,7 @@ t_trace_ip_address(_Config) -> ?assertEqual([], emqx_trace_handler:running()). filesync(Name, Type) -> + ct:sleep(50), filesync(Name, Type, 3). %% sometime the handler process is not started yet. diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index d1d3d16f8..7503f3b89 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,6 +1 @@ -# authentication: { -# mechanism: password-based -# backend: built-in-database -# user_id_type: clientid -# } - +authentication: [] diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index 1c5c00e85..b5dfa8f01 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -17,6 +17,8 @@ -ifndef(EMQX_AUTHN_HRL). -define(EMQX_AUTHN_HRL, true). +-include_lib("emqx/include/emqx_authentication.hrl"). + -define(APP, emqx_authn). -define(AUTHN, emqx_authentication). @@ -27,4 +29,9 @@ -define(AUTH_SHARD, emqx_authn_shard). +%% has to be the same as the root field name defined in emqx_schema +-define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME). +-define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). +-define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -endif. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index fbd31c5d2..5b3d76822 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -22,6 +22,8 @@ , check_configs/1 ]). +-include("emqx_authn.hrl"). + providers() -> [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} , {{'password-based', mysql}, emqx_authn_mysql} @@ -44,8 +46,8 @@ check_config(Config) -> check_config(Config, Opts) -> case do_check_config(Config, Opts) of - #{config := Checked} -> Checked; - #{<<"config">> := WithDefaults} -> WithDefaults + #{?CONF_NS_ATOM := Checked} -> Checked; + #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> @@ -56,10 +58,15 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> case lists:keyfind(Key, 1, providers()) of false -> throw({unknown_handler, Key}); - {_, Provider} -> - hocon_schema:check_plain(Provider, #{<<"config">> => Config}, + {_, ProviderModule} -> + hocon_schema:check_plain(ProviderModule, #{?CONF_NS_BINARY => Config}, Opts#{atom_key => true}) end. atom(Bin) -> - binary_to_existing_atom(Bin, utf8). + try + binary_to_existing_atom(Bin, utf8) + catch + _ : _ -> + throw({unknown_auth_provider, Bin}) + end. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index b83d9d1af..595eed1c1 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,6 +22,7 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -import(hoconsc, [mk/2, ref/1]). -import(emqx_dashboard_swagger, [error_codes/2]). @@ -32,8 +33,10 @@ % Swagger --define(API_TAGS_GLOBAL, [<<"authentication">>, <<"authentication config(global)">>]). --define(API_TAGS_SINGLE, [<<"authentication">>, <<"authentication config(single listener)">>]). +-define(API_TAGS_GLOBAL, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(global)">>]). +-define(API_TAGS_SINGLE, [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY, + <<"authentication config(single listener)">>]). -export([ api_spec/0 , paths/0 @@ -793,9 +796,10 @@ add_user(ChainName, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), - case emqx_authentication:add_user(ChainName, AuthenticatorID, #{ user_id => UserID - , password => Password - , is_superuser => IsSuperuser}) of + case emqx_authentication:add_user(ChainName, AuthenticatorID, + #{ user_id => UserID + , password => Password + , is_superuser => IsSuperuser}) of {ok, User} -> {201, User}; {error, Reason} -> @@ -845,7 +849,8 @@ list_users(ChainName, AuthenticatorID, PageParams) -> end. update_config(Path, ConfigRequest) -> - emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true, + override_to => cluster}). get_raw_config_with_defaults(ConfKeyPath) -> NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], @@ -1027,7 +1032,7 @@ authenticator_examples() -> backend => <<"redis">>, server => <<"127.0.0.1:6379">>, database => 0, - query => <<"HMGET ${username} password_hash salt">>, + cmd => <<"HMGET ${username} password_hash salt">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> } diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index df7fbecd3..035e61910 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -25,6 +25,8 @@ , stop/1 ]). +-include_lib("emqx/include/emqx_authentication.hrl"). + -dialyzer({nowarn_function, [start/2]}). %%------------------------------------------------------------------------------ @@ -65,7 +67,7 @@ chain_configs() -> [global_chain_config() | listener_chain_configs()]. global_chain_config() -> - {?GLOBAL, emqx:get_raw_config([<<"authentication">>], [])}. + {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}. listener_chain_configs() -> lists:map( @@ -77,7 +79,7 @@ listener_chain_configs() -> auth_config_path(ListenerID) -> [<<"listeners">>] ++ binary:split(atom_to_binary(ListenerID), <<":">>) - ++ [<<"authentication">>]. + ++ [?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY]. provider_types() -> lists:map(fun({Type, _Module}) -> Type end, emqx_authn:providers()). diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 22f62f519..c2e963ec4 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -22,10 +22,12 @@ , roots/0 , fields/1 , authenticator_type/0 + , root_type/0 + , mechanism/1 + , backend/1 ]). -%% only for doc generation -roots() -> [{authenticator_config, hoconsc:mk(authenticator_type())}]. +roots() -> []. fields(_) -> []. @@ -35,6 +37,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(desc) -> "Set to false to disable this auth provider"; enable(_) -> undefined. authenticator_type() -> @@ -42,3 +45,18 @@ authenticator_type() -> config_refs(Modules) -> lists:append([Module:refs() || Module <- Modules]). + +%% authn is a core functionality however implemented outside fo emqx app +%% in emqx_schema, 'authentication' is a map() type which is to allow +%% EMQ X more plugable. +root_type() -> + T = authenticator_type(), + hoconsc:union([T, hoconsc:array(T)]). + +mechanism(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). + +backend(Name) -> + hoconsc:mk(hoconsc:enum([Name]), + #{nullable => false}). diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 56f485afc..2205d237d 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -93,6 +93,8 @@ is_superuser(#{<<"is_superuser">> := 0}) -> #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := null}) -> #{is_superuser => false}; +is_superuser(#{<<"is_superuser">> := undefined}) -> + #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := false}) -> #{is_superuser => false}; is_superuser(#{<<"is_superuser">> := _}) -> diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 5a477c7e0..3604455dc 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -83,11 +83,11 @@ mnesia(boot) -> namespace() -> "authn-scram-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, [scram]}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('scram')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). @@ -105,7 +105,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{algorithm := Algorithm, @@ -137,10 +137,7 @@ authenticate(_Credential, _State) -> ignore. destroy(#{user_group := UserGroup}) -> - MatchSpec = ets:fun2ms( - fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup -> - User - end), + MatchSpec = group_match_spec(UserGroup), trans( fun() -> ok = lists:foreach(fun(UserInfo) -> @@ -205,16 +202,16 @@ lookup_user(UserID, #{user_group := UserGroup}) -> end. list_users(PageParams, #{user_group := UserGroup}) -> - MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_', '_', '_'}, [], ['$_']}], + MatchSpec = group_match_spec(UserGroup), {ok, emqx_mgmt_api:paginate(?TAB, MatchSpec, PageParams, ?FORMAT_FUN)}. %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ -ensure_auth_method('SCRAM-SHA-256', #{algorithm := sha256}) -> +ensure_auth_method(<<"SCRAM-SHA-256">>, #{algorithm := sha256}) -> true; -ensure_auth_method('SCRAM-SHA-512', #{algorithm := sha512}) -> +ensure_auth_method(<<"SCRAM-SHA-512">>, #{algorithm := sha512}) -> true; ensure_auth_method(_, _) -> false. @@ -228,8 +225,10 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S #{iteration_count => IterationCount, retrieve => RetrieveFun} ) of - {cotinue, ServerFirstMessage, Cache} -> - {cotinue, ServerFirstMessage, Cache}; + {continue, ServerFirstMessage, Cache} -> + {continue, ServerFirstMessage, Cache}; + ignore -> + ignore; {error, _Reason} -> {error, not_authorized} end. @@ -280,3 +279,9 @@ trans(Fun, Args) -> format_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> #{user_id => UserID, is_superuser => IsSuperuser}. + +group_match_spec(UserGroup) -> + ets:fun2ms( + fun(#user_info{user_id = {Group, _}} = User) when Group =:= UserGroup -> + User + end). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index ec2da3237..533301ea7 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -43,8 +43,9 @@ namespace() -> "authn-http". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(get) -> @@ -60,8 +61,8 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {mechanism, hoconsc:enum(['password-based'])} - , {backend, hoconsc:enum(['http'])} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(http)} , {url, fun url/1} , {body, fun body/1} , {request_timeout, fun request_timeout/1} @@ -233,9 +234,9 @@ transform_header_name(Headers) -> end, #{}, Headers). check_ssl_opts(Conf) -> - case parse_url(hocon_schema:get_value("config.url", Conf)) of + case parse_url(get_conf_val("url", Conf)) of #{scheme := https} -> - case hocon_schema:get_value("config.ssl.enable", Conf) of + case get_conf_val("ssl.enable", Conf) of true -> ok; false -> false end; @@ -244,8 +245,8 @@ check_ssl_opts(Conf) -> end. check_headers(Conf) -> - Method = to_bin(hocon_schema:get_value("config.method", Conf)), - Headers = hocon_schema:get_value("config.headers", Conf), + Method = to_bin(get_conf_val("method", Conf)), + Headers = get_conf_val("headers", Conf), Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). parse_url(URL) -> @@ -284,7 +285,7 @@ replace_placeholders([{K, V0} | More], Credential, Acc) -> undefined -> error({cannot_get_variable, V0}); V -> - replace_placeholders(More, Credential, [{K, emqx_authn_utils:bin(V)} | Acc]) + replace_placeholders(More, Credential, [{K, to_bin(V)} | Acc]) end. append_query(Path, []) -> @@ -340,3 +341,6 @@ to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L). + +get_conf_val(Name, Conf) -> + hocon_schema:get_value(?CONF_NS ++ "." ++ Name, Conf). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index d8ceb7f40..3fc4bac13 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -20,6 +20,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("jose/include/jose_jwk.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + -export([ start_link/1 , stop/1 @@ -66,9 +68,9 @@ init([Opts]) -> handle_call(get_cached_jwks, _From, #{jwks := Jwks} = State) -> {reply, {ok, Jwks}, State}; -handle_call({update, Opts}, _From, State) -> - State = handle_options(Opts), - {reply, ok, refresh_jwks(State)}; +handle_call({update, Opts}, _From, _State) -> + NewState = handle_options(Opts), + {reply, ok, refresh_jwks(NewState)}; handle_call(_Req, _From, State) -> {reply, ok, State}. @@ -91,25 +93,27 @@ handle_info({refresh_jwks, _TRef, refresh}, #{request_id := RequestID} = State) handle_info({http, {RequestID, Result}}, #{request_id := RequestID, endpoint := Endpoint} = State0) -> + ?tp(debug, jwks_endpoint_response, #{request_id => RequestID}), State1 = State0#{request_id := undefined}, - case Result of - {error, Reason} -> - ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", - endpoint => Endpoint, - reason => Reason}), - State1; - {_StatusLine, _Headers, Body} -> - try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), - {_, JWKs} = JWKS#jose_jwk.keys, - State1#{jwks := JWKs} - catch _:_ -> - ?SLOG(warning, #{msg => "invalid_jwks_returned", - endpoint => Endpoint, - body => Body}), - State1 - end - end; + NewState = case Result of + {error, Reason} -> + ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", + endpoint => Endpoint, + reason => Reason}), + State1; + {_StatusLine, _Headers, Body} -> + try + JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + {_, JWKs} = JWKS#jose_jwk.keys, + State1#{jwks := JWKs} + catch _:_ -> + ?SLOG(warning, #{msg => "invalid_jwks_returned", + endpoint => Endpoint, + body => Body}), + State1 + end + end, + {noreply, NewState}; handle_info({http, {_, _}}, State) -> %% ignore @@ -147,17 +151,18 @@ refresh_jwks(#{endpoint := Endpoint, NState = case httpc:request(get, {Endpoint, [{"Accept", "application/json"}]}, HTTPOpts, [{body_format, binary}, {sync, false}, {receiver, self()}]) of {error, Reason} -> - ?SLOG(warning, #{msg => "failed_to_request_jwks_endpoint", - endpoint => Endpoint, - reason => Reason}), + ?tp(warning, jwks_endpoint_request_fail, #{endpoint => Endpoint, + http_opts => HTTPOpts, + reason => Reason}), State; {ok, RequestID} -> + ?tp(debug, jwks_endpoint_request_ok, #{request_id => RequestID}), State#{request_id := RequestID} end, ensure_expiry_timer(NState). ensure_expiry_timer(State = #{refresh_interval := Interval}) -> - State#{refresh_timer := emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}. + State#{refresh_timer => emqx_misc:start_timer(timer:seconds(Interval), refresh_jwks)}. cancel_timer(State = #{refresh_timer := undefined}) -> State; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 7ec7eac6d..9295e5c7e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -16,6 +16,7 @@ -module(emqx_authn_jwt). +-include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -40,9 +41,9 @@ namespace() -> "authn-jwt". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{} - )} + [ {?CONF_NS, + hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields('hmac-based') -> @@ -82,7 +83,7 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {mechanism, {enum, [jwt]}} + [ {mechanism, emqx_authn_schema:mechanism('jwt')} , {verify_claims, fun verify_claims/1} ] ++ emqx_authn_schema:common_fields(). @@ -157,7 +158,7 @@ update(#{use_jwks := false} = Config, _State) -> update(#{use_jwks := true} = Config, #{jwk := Connector} = State) when is_pid(Connector) -> - ok = emqx_authn_jwks_connector:update(Connector, Config), + ok = emqx_authn_jwks_connector:update(Connector, connector_opts(Config)), case maps:get(verify_cliams, Config, undefined) of undefined -> {ok, State}; @@ -208,7 +209,7 @@ create2(#{use_jwks := false, JWK = jose_jwk:from_oct(Secret), {ok, #{jwk => JWK, verify_claims => VerifyClaims}} - end; + end; create2(#{use_jwks := false, algorithm := 'public-key', @@ -219,13 +220,8 @@ create2(#{use_jwks := false, verify_claims => VerifyClaims}}; create2(#{use_jwks := true, - verify_claims := VerifyClaims, - ssl := #{enable := Enable} = SSL} = Config) -> - SSLOpts = case Enable of - true -> maps:without([enable], SSL); - false -> #{} - end, - case emqx_authn_jwks_connector:start_link(Config#{ssl_opts => SSLOpts}) of + verify_claims := VerifyClaims} = Config) -> + case emqx_authn_jwks_connector:start_link(connector_opts(Config)) of {ok, Connector} -> {ok, #{jwk => Connector, verify_claims => VerifyClaims}}; @@ -233,6 +229,14 @@ create2(#{use_jwks := true, {error, Reason} end. +connector_opts(#{ssl := #{enable := Enable} = SSL} = Config) -> + SSLOpts = case Enable of + true -> maps:without([enable], SSL); + false -> #{} + end, + Config#{ssl_opts => SSLOpts}. + + may_decode_secret(false, Secret) -> Secret; may_decode_secret(true, Secret) -> try base64:decode(Secret) @@ -260,7 +264,7 @@ verify(JWS, [JWK | More], VerifyClaims) -> Claims = emqx_json:decode(Payload, [return_maps]), case verify_claims(Claims, VerifyClaims) of ok -> - {ok, #{is_superuser => maps:get(<<"is_superuser">>, Claims, false)}}; + {ok, emqx_authn_utils:is_superuser(Claims)}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index fd02671fb..f609d8cac 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -85,11 +85,11 @@ mnesia(boot) -> namespace() -> "authn-builtin_db". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, ['built-in-database']}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend('built-in-database')} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} ] ++ emqx_authn_schema:common_fields(); @@ -104,7 +104,7 @@ fields(other_algorithms) -> ]. user_id_type(type) -> user_id_type(); -user_id_type(default) -> username; +user_id_type(default) -> <<"username">>; user_id_type(_) -> undefined. password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), @@ -121,7 +121,7 @@ salt_rounds(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(AuthenticatorID, #{user_id_type := Type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 40bd0c2c9..3b47bcd7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -42,8 +42,8 @@ namespace() -> "authn-mongodb". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,8 +56,8 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mongodb]}} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mongodb)} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} @@ -115,6 +115,8 @@ create(#{selector := Selector} = Config) -> password_hash_algorithm, salt_position], Config), + #{password_hash_algorithm := Algorithm} = State, + ok = emqx_authn_utils:ensure_apps_started(Algorithm), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), NState = State#{ selector => NSelector, @@ -155,7 +157,7 @@ authenticate(#{password := Password} = Credential, Doc -> case check_password(Password, Doc, State) of ok -> - {ok, #{is_superuser => is_superuser(Doc, State)}}; + {ok, is_superuser(Doc, State)}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> ?SLOG(error, #{msg => "cannot_find_password_hash_field", resource => ResourceId, @@ -234,9 +236,10 @@ check_password(Password, end. is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> - maps:get(IsSuperuserField, Doc, false); + IsSuperuser = maps:get(IsSuperuserField, Doc, false), + emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser}); is_superuser(_, _) -> - false. + emqx_authn_utils:is_superuser(#{<<"is_superuser">> => false}). hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 47ca0ae3c..fd0d09f57 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -41,11 +41,11 @@ namespace() -> "authn-mysql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [mysql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(mysql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -74,7 +74,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 660acf566..fdd30b618 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -47,11 +47,11 @@ namespace() -> "authn-postgresql". -roots() -> [config]. +roots() -> [?CONF_NS]. -fields(config) -> - [ {mechanism, {enum, ['password-based']}} - , {backend, {enum, [postgresql]}} +fields(?CONF_NS) -> + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(postgresql)} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} @@ -75,7 +75,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, config)]. + [hoconsc:ref(?MODULE, ?CONF_NS)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 963536e0b..e17d0ad8f 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -42,8 +42,8 @@ namespace() -> "authn-redis". roots() -> - [ {config, hoconsc:mk(hoconsc:union(refs()), - #{})} + [ {?CONF_NS, hoconsc:mk(hoconsc:union(refs()), + #{})} ]. fields(standalone) -> @@ -56,15 +56,15 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [{mechanism, {enum, ['password-based']}}, - {backend, {enum, [redis]}}, - {query, fun query/1}, - {password_hash_algorithm, fun password_hash_algorithm/1}, - {salt_position, fun salt_position/1} + [ {mechanism, emqx_authn_schema:mechanism('password-based')} + , {backend, emqx_authn_schema:backend(redis)} + , {cmd, fun cmd/1} + , {password_hash_algorithm, fun password_hash_algorithm/1} + , {salt_position, fun salt_position/1} ] ++ emqx_authn_schema:common_fields(). -query(type) -> string(); -query(_) -> undefined. +cmd(type) -> string(); +cmd(_) -> undefined. password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; password_hash_algorithm(default) -> sha256; @@ -87,17 +87,17 @@ refs() -> create(_AuthenticatorID, Config) -> create(Config). -create(#{query := Query, +create(#{cmd := Cmd, password_hash_algorithm := Algorithm} = Config) -> try - NQuery = parse_query(Query), + NCmd = parse_cmd(Cmd), ok = emqx_authn_utils:ensure_apps_started(Algorithm), State = maps:with( [password_hash_algorithm, salt_position], Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), NState = State#{ - query => NQuery, + cmd => NCmd, resource_id => ResourceId}, case emqx_resource:create_local(ResourceId, emqx_connector_redis, Config) of {ok, already_created} -> @@ -108,8 +108,8 @@ create(#{query := Query, {error, Reason} end catch - error:{unsupported_query, _Query} -> - {error, {unsupported_query, Query}}; + error:{unsupported_cmd, _Cmd} -> + {error, {unsupported_cmd, Cmd}}; error:missing_password_hash -> {error, missing_password_hash}; error:{unsupported_fields, Fields} -> @@ -128,7 +128,7 @@ update(Config, State) -> authenticate(#{auth_method := _}, _) -> ignore; authenticate(#{password := Password} = Credential, - #{query := {Command, Key, Fields}, + #{cmd := {Command, Key, Fields}, resource_id := ResourceId} = State) -> NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), case emqx_resource:query(ResourceId, {cmd, [Command, NKey | Fields]}) of @@ -162,15 +162,15 @@ destroy(#{resource_id := ResourceId}) -> %%------------------------------------------------------------------------------ %% Only support HGET and HMGET -parse_query(Query) -> - case string:tokens(Query, " ") of +parse_cmd(Cmd) -> + case string:tokens(Cmd, " ") of [Command, Key, Field | Fields] when Command =:= "HGET" orelse Command =:= "HMGET" -> NFields = [Field | Fields], check_fields(NFields), NKey = parse_key(Key), {Command, NKey, NFields}; _ -> - error({unsupported_query, Query}) + error({unsupported_cmd, Cmd}) end. check_fields(Fields) -> diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl deleted file mode 100644 index d3704679f..000000000 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ /dev/null @@ -1,22 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authn_SUITE). - --compile(export_all). --compile(nowarn_export_all). - -all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index c93bec582..885811fec 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -43,18 +43,20 @@ groups() -> []. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( - [authentication], + [?CONF_NS_ATOM], ?GLOBAL), emqx_authn_test_lib:delete_authenticators( - [listeners, tcp, default, authentication], + [listeners, tcp, default, ?CONF_NS_ATOM], ?TCP_DEFAULT), {atomic, ok} = mria:clear_table(emqx_authn_mnesia), Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), ok = emqx_common_test_helpers:start_apps( [emqx_authn, emqx_dashboard], fun set_special_configs/1), @@ -87,8 +89,8 @@ set_special_configs(_App) -> %%------------------------------------------------------------------------------ t_invalid_listener(_) -> - {ok, 404, _} = request(get, uri(["listeners", "invalid", "authentication"])), - {ok, 404, _} = request(get, uri(["listeners", "in:valid", "authentication"])). + {ok, 404, _} = request(get, uri(["listeners", "invalid", ?CONF_NS])), + {ok, 404, _} = request(get, uri(["listeners", "in:valid", ?CONF_NS])). t_authenticators(_) -> test_authenticators([]). @@ -131,86 +133,86 @@ test_authenticators(PathPrefix) -> ValidConfig = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig), {ok, 409, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig), InvalidConfig0 = ValidConfig#{method => <<"delete">>}, {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), InvalidConfig0), InvalidConfig1 = ValidConfig#{method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>}}, {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), InvalidConfig1), ?assertAuthenticatorsMatch( [#{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}], - PathPrefix ++ ["authentication"]). + PathPrefix ++ [?CONF_NS]). test_authenticator(PathPrefix) -> ValidConfig0 = emqx_authn_test_lib:http_example(), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), ValidConfig0), {ok, 200, _} = request( get, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"])), {ok, 404, _} = request( get, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])), {ok, 404, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:built-in-database"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database"]), emqx_authn_test_lib:built_in_database_example()), InvalidConfig0 = ValidConfig0#{method => <<"delete">>}, {ok, 400, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), InvalidConfig0), InvalidConfig1 = ValidConfig0#{method => <<"get">>, headers => #{<<"content-type">> => <<"application/json">>}}, {ok, 400, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), InvalidConfig1), ValidConfig1 = ValidConfig0#{pool_size => 9}, {ok, 200, _} = request( put, - uri(PathPrefix ++ ["authentication", "password-based:http"]), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"]), ValidConfig1), {ok, 404, _} = request( delete, - uri(PathPrefix ++ ["authentication", "password-based:redis"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:redis"])), {ok, 204, _} = request( delete, - uri(PathPrefix ++ ["authentication", "password-based:http"])), + uri(PathPrefix ++ [?CONF_NS, "password-based:http"])), - ?assertAuthenticatorsMatch([], PathPrefix ++ ["authentication"]). + ?assertAuthenticatorsMatch([], PathPrefix ++ [?CONF_NS]). test_authenticator_users(PathPrefix) -> - UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), InvalidUsers = [ @@ -261,11 +263,11 @@ test_authenticator_users(PathPrefix) -> lists:usort([ UserId || #{<<"user_id">> := UserId} <- Page1Users ++ Page2Users])). test_authenticator_user(PathPrefix) -> - UsersUri = uri(PathPrefix ++ ["authentication", "password-based:built-in-database", "users"]), + UsersUri = uri(PathPrefix ++ [?CONF_NS, "password-based:built-in-database", "users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), User = #{user_id => <<"u1">>, password => <<"p1">>}, @@ -309,7 +311,7 @@ test_authenticator_move(PathPrefix) -> fun(Conf) -> {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), Conf) end, AuthenticatorConfs), @@ -320,40 +322,40 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"jwt">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), % Invalid moves {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"up">>}), {ok, 400, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:invalid">>}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:redis">>}), {ok, 404, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:redis">>}), % Valid moves {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"top">>}), ?assertAuthenticatorsMatch( @@ -362,11 +364,11 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"http">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"bottom">>}), ?assertAuthenticatorsMatch( @@ -375,11 +377,11 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>}, #{<<"mechanism">> := <<"jwt">>} ], - PathPrefix ++ ["authentication"]), + PathPrefix ++ [?CONF_NS]), {ok, 204, _} = request( post, - uri(PathPrefix ++ ["authentication", "jwt", "move"]), + uri(PathPrefix ++ [?CONF_NS, "jwt", "move"]), #{position => <<"before:password-based:built-in-database">>}), ?assertAuthenticatorsMatch( @@ -388,17 +390,17 @@ test_authenticator_move(PathPrefix) -> #{<<"mechanism">> := <<"jwt">>}, #{<<"mechanism">> := <<"password-based">>, <<"backend">> := <<"built-in-database">>} ], - PathPrefix ++ ["authentication"]). + PathPrefix ++ [?CONF_NS]). test_authenticator_import_users(PathPrefix) -> ImportUri = uri( PathPrefix ++ - ["authentication", "password-based:built-in-database", "import_users"]), + [?CONF_NS, "password-based:built-in-database", "import_users"]), {ok, 200, _} = request( post, - uri(PathPrefix ++ ["authentication"]), + uri(PathPrefix ++ [?CONF_NS]), emqx_authn_test_lib:built_in_database_example()), {ok, 400, _} = request(post, ImportUri, #{}), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 4a966fe0e..2c0716e8b 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -24,7 +24,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --define(PATH, [authentication]). +-define(PATH, [?CONF_NS_ATOM]). -define(HTTP_PORT, 33333). -define(HTTP_PATH, "/auth"). @@ -39,6 +39,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), application:ensure_all_started(cowboy), Config. @@ -52,6 +53,7 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authn_test_lib:delete_authenticators( [authentication], ?GLOBAL), diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 54db0a3c5..0a166616a 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -21,15 +21,25 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_authn.hrl"). -define(AUTHN_ID, <<"mechanism:jwt">>). +-define(JWKS_PORT, 33333). +-define(JWKS_PATH, "/jwks.json"). + + all() -> emqx_common_test_helpers:all(?MODULE). +init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + Config. + init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), Config. @@ -37,7 +47,11 @@ end_per_suite(_) -> emqx_common_test_helpers:stop_apps([emqx_authn]), ok. -t_jwt_authenticator(_) -> +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_jwt_authenticator_hmac_based(_) -> Secret = <<"abcdef">>, Config = #{mechanism => jwt, use_jwks => false, @@ -121,10 +135,9 @@ t_jwt_authenticator(_) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State3)), ok. -t_jwt_authenticator2(_) -> - Dir = code:lib_dir(emqx_authn, test), - PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), - PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), +t_jwt_authenticator_public_key(_) -> + PublicKey = test_rsa_key(public), + PrivateKey = test_rsa_key(private), Config = #{mechanism => jwt, use_jwks => false, algorithm => 'public-key', @@ -142,6 +155,78 @@ t_jwt_authenticator2(_) -> ?assertEqual(ok, emqx_authn_jwt:destroy(State)), ok. +t_jwks_renewal(_Config) -> + ok = emqx_authn_http_test_server:start(?JWKS_PORT, ?JWKS_PATH), + ok = emqx_authn_http_test_server:set_handler(fun jwks_handler/2), + + PrivateKey = test_rsa_key(private), + Payload = #{<<"username">> => <<"myuser">>}, + JWS = generate_jws('public-key', Payload, PrivateKey), + Credential = #{username => <<"myuser">>, + password => JWS}, + + BadConfig = #{mechanism => jwt, + algorithm => 'public-key', + ssl => #{enable => false}, + verify_claims => [], + + use_jwks => true, + endpoint => "http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT + 1) ++ ?JWKS_PATH, + refresh_interval => 1000 + }, + + ok = snabbkaffe:start_trace(), + + {{ok, State0}, _} = ?wait_async_action( + emqx_authn_jwt:create(?AUTHN_ID, BadConfig), + #{?snk_kind := jwks_endpoint_response}, + 1000), + + ok = snabbkaffe:stop(), + + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential, State0)), + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State0)), + + GoodConfig = BadConfig#{endpoint => + "http://127.0.0.1:" ++ integer_to_list(?JWKS_PORT) ++ ?JWKS_PATH}, + + ok = snabbkaffe:start_trace(), + + {{ok, State1}, _} = ?wait_async_action( + emqx_authn_jwt:update(GoodConfig, State0), + #{?snk_kind := jwks_endpoint_response}, + 1000), + + ok = snabbkaffe:stop(), + + ?assertEqual({ok, #{is_superuser => false}}, emqx_authn_jwt:authenticate(Credential, State1)), + ?assertEqual(ignore, emqx_authn_jwt:authenticate(Credential#{password => <<"badpassword">>}, State1)), + + ?assertEqual(ok, emqx_authn_jwt:destroy(State1)), + ok = emqx_authn_http_test_server:stop(). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +jwks_handler(Req0, State) -> + JWK = jose_jwk:from_pem_file(test_rsa_key(public)), + JWKS = jose_jwk_set:to_map([JWK], #{}), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + jiffy:encode(JWKS), + Req0), + {ok, Req, State}. + +test_rsa_key(public) -> + Dir = code:lib_dir(emqx_authn, test), + list_to_binary(filename:join([Dir, "data/public_key.pem"])); + +test_rsa_key(private) -> + Dir = code:lib_dir(emqx_authn, test), + list_to_binary(filename:join([Dir, "data/private_key.pem"])). + generate_jws('hmac-based', Payload, Secret) -> JWK = jose_jwk:from_oct(Secret), Header = #{ <<"alg">> => <<"HS256">> diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index b5bca513c..53c8016cd 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -29,6 +29,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + _ = application:load(emqx_conf), emqx_common_test_helpers:start_apps([emqx_authn]), Config. @@ -37,7 +38,8 @@ end_per_suite(_) -> ok. init_per_testcase(_Case, Config) -> - mnesia:clear_table(emqx_authn_mnesia), + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + mria:clear_table(emqx_authn_mnesia), Config. end_per_testcase(_Case, Config) -> @@ -47,6 +49,8 @@ end_per_testcase(_Case, Config) -> %% Tests %%------------------------------------------------------------------------------ +-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}). + t_check_schema(_Config) -> ConfigOk = #{ <<"mechanism">> => <<"password-based">>, @@ -58,7 +62,7 @@ t_check_schema(_Config) -> } }, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigOk}), + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)), ConfigNotOk = #{ <<"mechanism">> => <<"password-based">>, @@ -72,7 +76,7 @@ t_check_schema(_Config) -> ?assertException( throw, {emqx_authn_mnesia, _}, - hocon_schema:check_plain(emqx_authn_mnesia, #{<<"config">> => ConfigNotOk})). + hocon_schema:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))). t_create(_) -> Config0 = config(), diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl new file mode 100644 index 000000000..562c5aa1b --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl @@ -0,0 +1,409 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mongo_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authn.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + + +-define(MONGO_HOST, "mongo"). +-define(MONGO_PORT, 27017). +-define(MONGO_CLIENT, 'emqx_authn_mongo_SUITE_client'). + +-define(PATH, [authentication]). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_testcase(_TestCase, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + emqx_authentication:initialize_authentication(?GLOBAL, []), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + {ok, _} = mc_worker_api:connect(mongo_config()), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok = mc_worker_api:disconnect(?MONGO_CLIENT). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + case emqx_authn_test_lib:is_tcp_server_available(?MONGO_HOST, ?MONGO_PORT) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + ok = start_apps([emqx_resource, emqx_connector]), + Config; + false -> + {skip, no_mongo} + end. + +end_per_suite(_Config) -> + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + ok = stop_apps([emqx_resource, emqx_connector]), + ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + AuthConfig = raw_mongo_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + {ok, [#{provider := emqx_authn_mongodb}]} = emqx_authentication:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + AuthConfig = raw_mongo_auth_config(), + + InvalidConfigs = + [ + AuthConfig#{mongo_type => <<"unknown">>}, + AuthConfig#{selector => <<"{ \"username\": \"${username}\" }">>} + ], + + lists:foreach( + fun(Config) -> + {error, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + {ok, []} = emqx_authentication:list_authenticators(?GLOBAL) + end, + InvalidConfigs). + +t_authenticate(_Config) -> + ok = init_seeds(), + ok = lists:foreach( + fun(Sample) -> + ct:pal("test_user_auth sample: ~p", [Sample]), + test_user_auth(Sample) + end, + user_seeds()), + ok = drop_seeds(). + +test_user_auth(#{credentials := Credentials0, + config_params := SpecificConfigParams, + result := Result}) -> + AuthConfig = maps:merge(raw_mongo_auth_config(), SpecificConfigParams), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + Credentials = Credentials0#{ + listener => 'tcp:default', + protocol => mqtt + }, + ?assertEqual(Result, emqx_access_control:authenticate(Credentials)), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL). + +t_destroy(_Config) -> + ok = init_seeds(), + AuthConfig = raw_mongo_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig}), + + {ok, [#{provider := emqx_authn_mongodb, state := State}]} + = emqx_authentication:list_authenticators(?GLOBAL), + + {ok, _} = emqx_authn_mongodb:authenticate( + #{username => <<"plain">>, + password => <<"plain">> + }, + State), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + + % Authenticator should not be usable anymore + ?assertException( + error, + _, + emqx_authn_mongodb:authenticate( + #{username => <<"plain">>, + password => <<"plain">> + }, + State)), + + ok = drop_seeds(). + +t_update(_Config) -> + ok = init_seeds(), + CorrectConfig = raw_mongo_auth_config(), + IncorrectConfig = + CorrectConfig#{selector => #{<<"wrongfield">> => <<"wrongvalue">>}}, + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, IncorrectConfig}), + + {error, not_authorized} = emqx_access_control:authenticate( + #{username => <<"plain">>, + password => <<"plain">>, + listener => 'tcp:default', + protocol => mqtt + }), + + % We update with config with correct selector, provider should update and work properly + {ok, _} = emqx:update_config( + ?PATH, + {update_authenticator, ?GLOBAL, <<"password-based:mongodb">>, CorrectConfig}), + + {ok,_} = emqx_access_control:authenticate( + #{username => <<"plain">>, + password => <<"plain">>, + listener => 'tcp:default', + protocol => mqtt + }), + ok = drop_seeds(). + +t_is_superuser(_Config) -> + Config = raw_mongo_auth_config(), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + Checks = [ + {<<"0">>, false}, + {<<"">>, false}, + {null, false}, + {false, false}, + {0, false}, + + {<<"1">>, true}, + {<<"val">>, true}, + {1, true}, + {123, true}, + {true, true} + ], + + lists:foreach(fun test_is_superuser/1, Checks). + +test_is_superuser({Value, ExpectedValue}) -> + {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}), + + UserData = #{ + username => <<"user">>, + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => Value + }, + + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, [UserData]), + + Credentials = #{ + listener => 'tcp:default', + protocol => mqtt, + username => <<"user">>, + password => <<"plain">> + }, + + ?assertEqual( + {ok, #{is_superuser => ExpectedValue}}, + emqx_access_control:authenticate(Credentials)). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +raw_mongo_auth_config() -> + #{ + mechanism => <<"password-based">>, + password_hash_algorithm => <<"plain">>, + salt_position => <<"suffix">>, + enable => <<"true">>, + + backend => <<"mongodb">>, + mongo_type => <<"single">>, + database => <<"mqtt">>, + collection => <<"users">>, + server => mongo_server(), + + selector => #{<<"username">> => <<"${username}">>}, + password_hash_field => <<"password_hash">>, + salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">> + }. + +user_seeds() -> + [#{data => #{ + username => <<"plain">>, + password_hash => <<"plainsalt">>, + salt => <<"salt">>, + is_superuser => <<"1">> + }, + credentials => #{ + username => <<"plain">>, + password => <<"plain">> + }, + config_params => #{ + }, + result => {ok,#{is_superuser => true}} + }, + + #{data => #{ + username => <<"md5">>, + password_hash => <<"9b4d0c43d206d48279e69b9ad7132e22">>, + salt => <<"salt">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"md5">>, + password => <<"md5">> + }, + config_params => #{ + password_hash_algorithm => <<"md5">>, + salt_position => <<"suffix">> + }, + result => {ok,#{is_superuser => false}} + }, + + #{data => #{ + username => <<"sha256">>, + password_hash => <<"ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf">>, + salt => <<"salt">>, + is_superuser => 1 + }, + credentials => #{ + clientid => <<"sha256">>, + password => <<"sha256">> + }, + config_params => #{ + selector => #{<<"username">> => <<"${clientid}">>}, + password_hash_algorithm => <<"sha256">>, + salt_position => <<"prefix">> + }, + result => {ok,#{is_superuser => true}} + }, + + #{data => #{ + username => <<"bcrypt">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => 0 + }, + credentials => #{ + username => <<"bcrypt">>, + password => <<"bcrypt">> + }, + config_params => #{ + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> % should be ignored + }, + result => {ok,#{is_superuser => false}} + }, + + #{data => #{ + username => <<"bcrypt0">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt0">>, + password => <<"bcrypt">> + }, + config_params => #{ + % clientid variable & username credentials + selector => #{<<"username">> => <<"${clientid}">>}, + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,not_authorized} + }, + + #{data => #{ + username => <<"bcrypt1">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt1">>, + password => <<"bcrypt">> + }, + config_params => #{ + selector => #{<<"userid">> => <<"${clientid}">>}, + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,not_authorized} + }, + + #{data => #{ + username => <<"bcrypt2">>, + password_hash => <<"$2b$12$wtY3h20mUjjmeaClpqZVveDWGlHzCGsvuThMlneGHA7wVeFYyns2u">>, + salt => <<"$2b$12$wtY3h20mUjjmeaClpqZVve">>, + is_superuser => <<"0">> + }, + credentials => #{ + username => <<"bcrypt2">>, + % Wrong password + password => <<"wrongpass">> + }, + config_params => #{ + password_hash_algorithm => <<"bcrypt">>, + salt_position => <<"suffix">> + }, + result => {error,bad_username_or_password} + } + ]. + +init_seeds() -> + Users = [Values || #{data := Values} <- user_seeds()], + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"users">>, Users), + ok. + +drop_seeds() -> + {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"users">>, #{}), + ok. + +mongo_server() -> + iolist_to_binary( + io_lib:format( + "~s:~b", + [?MONGO_HOST, ?MONGO_PORT])). + +mongo_config() -> + [ + {database, <<"mqtt">>}, + {host, ?MONGO_HOST}, + {port, ?MONGO_PORT}, + {register, ?MONGO_CLIENT} + ]. + +start_apps(Apps) -> + lists:foreach(fun application:ensure_all_started/1, Apps). + +stop_apps(Apps) -> + lists:foreach(fun application:stop/1, Apps). diff --git a/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl new file mode 100644 index 000000000..a217cb96d --- /dev/null +++ b/apps/emqx_authn/test/emqx_authn_mqtt_test_client.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_mqtt_test_client). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx_mqtt.hrl"). + +%% API +-export([start_link/2, + stop/1]). + +-export([send/2]). + +%% gen_server callbacks + +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2]). + +-define(TIMEOUT, 1000). +-define(TCP_OPTIONS, [binary, {packet, raw}, {active, once}, + {nodelay, true}]). + +-define(PARSE_OPTIONS, + #{strict_mode => false, + max_size => ?MAX_PACKET_SIZE, + version => ?MQTT_PROTO_V5 + }). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +start_link(Host, Port) -> + gen_server:start_link(?MODULE, [Host, Port, self()], []). + +stop(Pid) -> + gen_server:call(Pid, stop). + +send(Pid, Packet) -> + gen_server:call(Pid, {send, Packet}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Host, Port, Owner]) -> + {ok, Socket} = gen_tcp:connect(Host, Port, ?TCP_OPTIONS, ?TIMEOUT), + {ok, #{owner => Owner, + socket => Socket, + parse_state => emqx_frame:initial_parse_state(?PARSE_OPTIONS) + }}. + +handle_info({tcp, _Sock, Data}, #{parse_state := PSt, + owner := Owner, + socket := Socket} = St) -> + {NewPSt, Packets} = process_incoming(PSt, Data, []), + ok = deliver(Owner, Packets), + ok = run_sock(Socket), + {noreply, St#{parse_state => NewPSt}}; + +handle_info({tcp_closed, _Sock}, St) -> + {stop, normal, St}. + +handle_call({send, Packet}, _From, #{socket := Socket} = St) -> + ok = gen_tcp:send(Socket, emqx_frame:serialize(Packet, ?MQTT_PROTO_V5)), + {reply, ok, St}; + +handle_call(stop, _From, #{socket := Socket} = St) -> + ok = gen_tcp:close(Socket), + {stop, normal, ok, St}. + +handle_cast(_, St) -> + {noreply, St}. + +terminate(_Reason, _St) -> + ok. + +%%-------------------------------------------------------------------- +%% internal functions +%%-------------------------------------------------------------------- + +process_incoming(PSt, Data, Packets) -> + case emqx_frame:parse(Data, PSt) of + {more, NewPSt} -> + {NewPSt, lists:reverse(Packets)}; + {ok, Packet, Rest, NewPSt} -> + process_incoming(NewPSt, Rest, [Packet | Packets]) + end. + +deliver(_Owner, []) -> ok; +deliver(Owner, [Packet | Packets]) -> + Owner ! {packet, Packet}, + deliver(Owner, Packets). + + +run_sock(Socket) -> + inet:setopts(Socket, [{active, once}]). diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl index 9073dd38a..bf66b034a 100644 --- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl @@ -38,6 +38,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?MYSQL_HOST, ?MYSQL_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), @@ -117,9 +119,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_mysql_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 08bb2ee2e..2a79179e1 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -38,6 +38,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy, t_is_superuser]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +54,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?PGSQL_HOST, ?PGSQL_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), @@ -117,9 +119,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_pgsql_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 8669080b0..2e941e72f 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -23,12 +23,10 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). - -define(REDIS_HOST, "redis"). -define(REDIS_PORT, 6379). -define(REDIS_RESOURCE, <<"emqx_authn_redis_SUITE">>). - -define(PATH, [authentication]). all() -> @@ -38,6 +36,7 @@ groups() -> [{require_seeds, [], [t_authenticate, t_update, t_destroy]}]. init_per_testcase(_, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), emqx_authentication:initialize_authentication(?GLOBAL, []), emqx_authn_test_lib:delete_authenticators( [authentication], @@ -53,6 +52,7 @@ end_per_group(require_seeds, Config) -> Config. init_per_suite(Config) -> + _ = application:load(emqx_conf), case emqx_authn_test_lib:is_tcp_server_available(?REDIS_HOST, ?REDIS_PORT) of true -> ok = emqx_common_test_helpers:start_apps([emqx_authn]), @@ -98,11 +98,11 @@ t_create_invalid(_Config) -> AuthConfig#{password => <<"wrongpass">>}, AuthConfig#{database => <<"5678">>}, AuthConfig#{ - query => <<"MGET password_hash:${username} salt:${username}">>}, + cmd => <<"MGET password_hash:${username} salt:${username}">>}, AuthConfig#{ - query => <<"HMGET mqtt_user:${username} password_hash invalid_field">>}, + cmd => <<"HMGET mqtt_user:${username} password_hash invalid_field">>}, AuthConfig#{ - query => <<"HMGET mqtt_user:${username} salt is_superuser">>} + cmd => <<"HMGET mqtt_user:${username} salt is_superuser">>} ], lists:foreach( @@ -124,9 +124,9 @@ t_authenticate(_Config) -> user_seeds()). test_user_auth(#{credentials := Credentials0, - config_params := SpecificConfgParams, + config_params := SpecificConfigParams, result := Result}) -> - AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfgParams), + AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams), {ok, _} = emqx:update_config( ?PATH, @@ -177,7 +177,7 @@ t_update(_Config) -> CorrectConfig = raw_redis_auth_config(), IncorrectConfig = CorrectConfig#{ - query => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>}, + cmd => <<"HMGET invalid_key:${username} password_hash salt is_superuser">>}, {ok, _} = emqx:update_config( ?PATH, @@ -214,7 +214,7 @@ raw_redis_auth_config() -> enable => <<"true">>, backend => <<"redis">>, - query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, database => <<"1">>, password => <<"public">>, server => redis_server() @@ -262,7 +262,7 @@ user_seeds() -> }, key => "mqtt_user:sha256", config_params => #{ - query => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${clientid} password_hash salt is_superuser">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> }, @@ -298,7 +298,7 @@ user_seeds() -> key => "mqtt_user:bcrypt0", config_params => #{ % clientid variable & username credentials - query => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_client:${clientid} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, @@ -316,8 +316,8 @@ user_seeds() -> }, key => "mqtt_user:bcrypt1", config_params => #{ - % Bad key in query - query => <<"HMGET badkey:${username} password_hash salt is_superuser">>, + % Bad key in cmd + cmd => <<"HMGET badkey:${username} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, @@ -336,7 +336,7 @@ user_seeds() -> }, key => "mqtt_user:bcrypt2", config_params => #{ - query => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, + cmd => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>, password_hash_algorithm => <<"bcrypt">>, salt_position => <<"suffix">> }, diff --git a/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl new file mode 100644 index 000000000..b63e6638a --- /dev/null +++ b/apps/emqx_authn/test/emqx_enhanced_authn_scram_mnesia_SUITE.erl @@ -0,0 +1,375 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_enhanced_authn_scram_mnesia_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include("emqx_authn.hrl"). + +-define(PATH, [authentication]). + +-define(USER_MAP, #{user_id := _, + is_superuser := _}). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + ok = emqx_common_test_helpers:start_apps([emqx_authn]), + Config. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_authn]). + +init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + mria:clear_table(emqx_enhanced_authn_scram_mnesia), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL), + Config. + +end_per_testcase(_Case, Config) -> + Config. + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + ValidConfig = #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => <<"sha512">>, + <<"iteration_count">> => <<"4096">> + }, + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, ValidConfig}), + + {ok, [#{provider := emqx_enhanced_authn_scram_mnesia}]} + = emqx_authentication:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + InvalidConfig = #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => <<"sha271828">>, + <<"iteration_count">> => <<"4096">> + }, + + {error, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, InvalidConfig}), + + {ok, []} = emqx_authentication:list_authenticators(?GLOBAL). + +t_authenticate(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage}) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => Password, + algorithm => Algorithm} + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage}), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage}) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => Algorithm} + ). + +t_authenticate_bad_username(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(<<"badusername">>), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_password(_Config) -> + Algorithm = sha512, + Username = <<"u">>, + Password = <<"p">>, + + init_auth(Username, Password, Algorithm), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + }), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage}) = receive_packet(), + + {continue, ClientFinalMessage, _ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => <<"badpassword">>, + algorithm => Algorithm} + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage}), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_destroy(_) -> + Config = config(), + OtherId = list_to_binary([<<"id-other">>]), + {ok, State0} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + {ok, StateOther} = emqx_enhanced_authn_scram_mnesia:create(OtherId, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State0), + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, StateOther), + + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State0), + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther), + + ok = emqx_enhanced_authn_scram_mnesia:destroy(State0), + + {ok, State1} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + {error,not_found} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State1), + {ok, _} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, StateOther). + +t_add_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + {error, already_exist} = emqx_enhanced_authn_scram_mnesia:add_user(User, State). + +t_delete_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + {error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State), + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + + ok = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State), + {error, not_found} = emqx_enhanced_authn_scram_mnesia:delete_user(<<"u">>, State). + +t_update_user(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + User = #{user_id => <<"u">>, password => <<"p">>}, + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(User, State), + {ok, #{is_superuser := false}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State), + + {ok, + #{user_id := <<"u">>, + is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:update_user( + <<"u">>, + #{password => <<"p1">>, is_superuser => true}, + State), + + {ok, #{is_superuser := true}} = emqx_enhanced_authn_scram_mnesia:lookup_user(<<"u">>, State). + +t_list_users(_) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + Users = [#{user_id => <<"u1">>, password => <<"p">>}, + #{user_id => <<"u2">>, password => <<"p">>}, + #{user_id => <<"u3">>, password => <<"p">>}], + + lists:foreach( + fun(U) -> {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(U, State) end, + Users), + + {ok, + #{data := [?USER_MAP, ?USER_MAP], + meta := #{page := 1, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users( + #{<<"page">> => 1, <<"limit">> => 2}, + State), + {ok, + #{data := [?USER_MAP], + meta := #{page := 2, limit := 2, count := 3}}} = emqx_enhanced_authn_scram_mnesia:list_users( + #{<<"page">> => 2, <<"limit">> => 2}, + State). + +t_is_superuser(_Config) -> + ok = test_is_superuser(#{is_superuser => false}, false), + ok = test_is_superuser(#{is_superuser => true}, true), + ok = test_is_superuser(#{}, false). + +test_is_superuser(UserInfo, ExpectedIsSuperuser) -> + Config = config(), + {ok, State} = emqx_enhanced_authn_scram_mnesia:create(<<"id">>, Config), + + Username = <<"u">>, + Password = <<"p">>, + + UserInfo0 = UserInfo#{user_id => Username, + password => Password}, + + {ok, _} = emqx_enhanced_authn_scram_mnesia:add_user(UserInfo0, State), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + {continue, ServerFirstMessage, ServerCache} + = emqx_enhanced_authn_scram_mnesia:authenticate( + #{auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFirstMessage, + auth_cache => #{} + }, + State), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{client_first_message => ClientFirstMessage, + password => Password, + algorithm => sha512} + ), + + {ok, UserInfo1, ServerFinalMessage} + = emqx_enhanced_authn_scram_mnesia:authenticate( + #{auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFinalMessage, + auth_cache => ServerCache + }, + State), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => sha512} + ), + + ?assertMatch(#{is_superuser := ExpectedIsSuperuser}, UserInfo1), + + ok = emqx_enhanced_authn_scram_mnesia:destroy(State). + + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +config() -> + #{ + mechanism => <<"scram">>, + backend => <<"built-in-database">>, + algorithm => sha512, + iteration_count => 4096 + }. + +raw_config(Algorithm) -> + #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"built-in-database">>, + <<"algorithm">> => atom_to_binary(Algorithm), + <<"iteration_count">> => <<"4096">> + }. + +init_auth(Username, Password, Algorithm) -> + Config = raw_config(Algorithm), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config}), + + {ok, [#{state := State}]} = emqx_authentication:list_authenticators(?GLOBAL), + + emqx_enhanced_authn_scram_mnesia:add_user( + #{user_id => Username, password => Password}, + State). + +receive_packet() -> + receive + {packet, Packet} -> + ct:pal("Delivered packet: ~p", [Packet]), + Packet + after 1000 -> + ct:fail("Deliver timeout") + end. diff --git a/apps/emqx_authz/README.md b/apps/emqx_authz/README.md index a44297a55..bda09481a 100644 --- a/apps/emqx_authz/README.md +++ b/apps/emqx_authz/README.md @@ -23,7 +23,7 @@ authz:{ keyfile: "etc/certs/client-key.pem" } } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}" }, { type: postgresql @@ -36,7 +36,7 @@ authz:{ auto_reconnect: true ssl: {enable: false} } - sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" }, { type: redis @@ -48,7 +48,7 @@ authz:{ auto_reconnect: true ssl: {enable: false} } - cmd: "HGETALL mqtt_authz:%u" + cmd: "HGETALL mqtt_authz:${username}" }, { principal: {username: "^admin?"} diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 3469aad3a..5bb6ab841 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -22,7 +22,7 @@ authorization { # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" # } - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}" # }, # { # type: postgresql @@ -33,7 +33,7 @@ authorization { # password: public # auto_reconnect: true # ssl: {enable: false} - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" # }, # { # type: redis @@ -43,7 +43,7 @@ authorization { # password: public # auto_reconnect: true # ssl: {enable: false} - # cmd: "HGETALL mqtt_authz:%u" + # cmd: "HGETALL mqtt_authz:${username}" # }, # { # type: mongodb @@ -53,7 +53,7 @@ authorization { # database: mqtt # ssl: {enable: false} # collection: mqtt_authz - # selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } + # selector: { "$or": [ { "username": "${username}" }, { "clientid": "${clientid}" } ] } # }, { type: built-in-database diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 4496e0299..0b5534608 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -40,6 +40,8 @@ -export([acl_conf_file/0]). +-export([ph_to_re/1]). + -spec(register_metrics() -> ok). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, ?AUTHZ_METRICS). @@ -64,11 +66,14 @@ move(Type, Cmd) -> move(Type, Cmd, #{}). move(Type, #{<<"before">> := Before}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts); move(Type, #{<<"after">> := After}, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts); move(Type, Position, Opts) -> - emqx:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}, Opts). + emqx:update_config( ?CONF_KEY_PATH + , {?CMD_MOVE, type(Type), Position}, Opts). update(Cmd, Sources) -> update(Cmd, Sources, #{}). @@ -155,7 +160,8 @@ do_post_update({{?CMD_REPLACE, Type}, Source}, _NewSources) when is_map(Source) {OldSource, Front, Rear} = take(Type, OldInitedSources), ok = ensure_resource_deleted(OldSource), InitedSources = init_sources(check_sources([Source])), - ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), + ok = emqx_hooks:put( 'client.authorize' + , {?MODULE, authorize, [Front ++ InitedSources ++ Rear]}, -1), ok = emqx_authz_cache:drain_cache(); do_post_update({{?CMD_DELETE, Type}, _Source}, _NewSources) -> OldInitedSources = lookup(), @@ -201,7 +207,12 @@ check_dup_types([Source | Sources], Checked) -> create_dry_run(T, Source) -> case is_connector_source(T) of true -> - [NSource] = check_sources([Source]), + [CheckedSource] = check_sources([Source]), + case T of + http -> + URIMap = maps:get(url, CheckedSource), + NSource = maps:put(base_url, maps:remove(query, URIMap), CheckedSource) + end, emqx_resource:create_dry_run(connector_module(T), NSource); false -> ok @@ -267,7 +278,7 @@ init_source(#{type := DB, {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id, - query => Mod:parse_query(SQL) + query => erlang:apply(Mod, parse_query, [SQL]) } } end. @@ -277,22 +288,36 @@ init_source(#{type := DB, %%-------------------------------------------------------------------- %% @doc Check AuthZ --spec(authorize(emqx_types:clientinfo(), emqx_types:all(), emqx_types:topic(), allow | deny, sources()) +-spec(authorize( emqx_types:clientinfo() + , emqx_types:all() + , emqx_types:topic() + , allow | deny + , sources()) -> {stop, allow} | {ok, deny}). authorize(#{username := Username, peerhost := IpAddress } = Client, PubSub, Topic, DefaultResult, Sources) -> case do_authorize(Client, PubSub, Topic, Sources) of {matched, allow} -> - ?SLOG(info, #{msg => "authorization_permission_allowed", username => Username, ipaddr => IpAddress, topic => Topic}), + ?SLOG(info, #{msg => "authorization_permission_allowed", + username => Username, + ipaddr => IpAddress, + topic => Topic}), emqx_metrics:inc(?AUTHZ_METRICS(allow)), {stop, allow}; {matched, deny} -> - ?SLOG(info, #{msg => "authorization_permission_denied", username => Username, ipaddr => IpAddress, topic => Topic}), + ?SLOG(info, #{msg => "authorization_permission_denied", + username => Username, + ipaddr => IpAddress, + topic => Topic}), emqx_metrics:inc(?AUTHZ_METRICS(deny)), {stop, deny}; nomatch -> - ?SLOG(info, #{msg => "authorization_failed_nomatch", username => Username, ipaddr => IpAddress, topic => Topic, reason => "no-match rule"}), + ?SLOG(info, #{msg => "authorization_failed_nomatch", + username => Username, + ipaddr => IpAddress, + topic => Topic, + reason => "no-match rule"}), {stop, DefaultResult} end. @@ -309,7 +334,7 @@ do_authorize(Client, PubSub, Topic, [#{type := file} = F | Tail]) -> do_authorize(Client, PubSub, Topic, [Connector = #{type := Type} | Tail] ) -> Mod = authz_module(Type), - case Mod:authorize(Client, PubSub, Topic, Connector) of + case erlang:apply(Mod, authorize, [Client, PubSub, Topic, Connector]) of nomatch -> do_authorize(Client, PubSub, Topic, Tail); Matched -> Matched end. @@ -381,8 +406,12 @@ type(postgresql) -> postgresql; type(<<"postgresql">>) -> postgresql; type('built-in-database') -> 'built-in-database'; type(<<"built-in-database">>) -> 'built-in-database'; -type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema +%% should never happend if the input is type-checked by hocon schema +type(Unknown) -> error({unknown_authz_source_type, Unknown}). %% @doc where the acl.conf file is stored. acl_conf_file() -> filename:join([emqx:data_dir(), "authz", "acl.conf"]). + +ph_to_re(VarPH) -> + re:replace(VarPH, "[\\$\\{\\}]", "\\\\&", [global, {return, list}]). diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 23c6077fa..df5a6c819 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -440,14 +440,15 @@ read_certs(#{<<"ssl">> := SSL} = Source) -> {error, Reason} -> ?SLOG(error, Reason#{msg => failed_to_readd_ssl_file}), throw(failed_to_readd_ssl_file); - NewSSL -> + {ok, NewSSL} -> Source#{<<"ssl">> => NewSSL} end; read_certs(Source) -> Source. maybe_write_certs(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> Type = maps:get(<<"type">>, Source), - emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL); + {ok, Return} = emqx_tls_lib:ensure_ssl_files(filename:join(["authz", Type]), SSL), + maps:put(<<"ssl">>, Return, Source); maybe_write_certs(Source) -> Source. write_file(Filename, Bytes0) -> diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 4c6af402c..6d1324c47 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -24,6 +24,7 @@ %% AuthZ Callbacks -export([ authorize/4 , description/0 + , parse_url/1 ]). -ifdef(TEST). @@ -36,7 +37,7 @@ description() -> authorize(Client, PubSub, Topic, #{type := http, - url := #{path := Path} = Url, + url := #{path := Path} = URL, headers := Headers, method := Method, request_timeout := RequestTimeout, @@ -44,7 +45,7 @@ authorize(Client, PubSub, Topic, } = Source) -> Request = case Method of get -> - Query = maps:get(query, Url, ""), + Query = maps:get(query, URL, ""), Path1 = replvar(Path ++ "?" ++ Query, PubSub, Topic, Client), {Path1, maps:to_list(Headers)}; _ -> @@ -56,10 +57,32 @@ authorize(Client, PubSub, Topic, Path1 = replvar(Path, PubSub, Topic, Client), {Path1, maps:to_list(Headers), Body1} end, - case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> {matched, allow}; - {ok, 200, _Headers, _Body} -> {matched, allow}; - _ -> nomatch + case emqx_resource:query(ResourceID, {Method, Request, RequestTimeout}) of + {ok, 200, _Headers} -> + {matched, allow}; + {ok, 204, _Headers} -> + {matched, allow}; + {ok, 200, _Headers, _Body} -> + {matched, allow}; + {ok, _Status, _Headers, _Body} -> + nomatch; + {error, Reason} -> + ?SLOG(error, #{msg => "http_server_query_failed", + resource => ResourceID, + reason => Reason}), + ignore + end. + +parse_url(URL) + when URL =:= undefined -> + #{}; +parse_url(URL) -> + {ok, URIMap} = emqx_http_lib:uri_parse(URL), + case maps:get(query, URIMap, undefined) of + undefined -> + URIMap#{query => ""}; + _ -> + URIMap end. query_string(Body) -> @@ -87,19 +110,19 @@ replvar(Str0, PubSub, Topic, }) when is_list(Str0); is_binary(Str0) -> NTopic = emqx_http_lib:uri_encode(Topic), - Str1 = re:replace( Str0, ?PH_S_CLIENTID - , Clientid, [global, {return, binary}]), - Str2 = re:replace( Str1, ?PH_S_USERNAME + Str1 = re:replace( Str0, emqx_authz:ph_to_re(?PH_S_CLIENTID) + , bin(Clientid), [global, {return, binary}]), + Str2 = re:replace( Str1, emqx_authz:ph_to_re(?PH_S_USERNAME) , bin(Username), [global, {return, binary}]), - Str3 = re:replace( Str2, ?PH_S_HOST + Str3 = re:replace( Str2, emqx_authz:ph_to_re(?PH_S_HOST) , inet_parse:ntoa(IpAddress), [global, {return, binary}]), - Str4 = re:replace( Str3, ?PH_S_PROTONAME + Str4 = re:replace( Str3, emqx_authz:ph_to_re(?PH_S_PROTONAME) , bin(Protocol), [global, {return, binary}]), - Str5 = re:replace( Str4, ?PH_S_MOUNTPOINT - , Mountpoint, [global, {return, binary}]), - Str6 = re:replace( Str5, ?PH_S_TOPIC - , NTopic, [global, {return, binary}]), - Str7 = re:replace( Str6, ?PH_S_ACTION + Str5 = re:replace( Str4, emqx_authz:ph_to_re(?PH_S_MOUNTPOINT) + , bin(Mountpoint), [global, {return, binary}]), + Str6 = re:replace( Str5, emqx_authz:ph_to_re(?PH_S_TOPIC) + , bin(NTopic), [global, {return, binary}]), + Str7 = re:replace( Str6, emqx_authz:ph_to_re(?PH_S_ACTION) , bin(PubSub), [global, {return, binary}]), Str7. diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index ec34a266c..5b55c23b7 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -76,11 +76,11 @@ replvar(Selector, #{clientid := Clientid, end || M <- V], AccIn); InFun(K, V, AccIn) when is_binary(V) -> - V1 = re:replace( V, ?PH_S_CLIENTID + V1 = re:replace( V, emqx_authz:ph_to_re(?PH_S_CLIENTID) , bin(Clientid), [global, {return, binary}]), - V2 = re:replace( V1, ?PH_S_USERNAME + V2 = re:replace( V1, emqx_authz:ph_to_re(?PH_S_USERNAME) , bin(Username), [global, {return, binary}]), - V3 = re:replace( V2, ?PH_S_HOST + V3 = re:replace( V2, emqx_authz:ph_to_re(?PH_S_HOST) , inet_parse:ntoa(IpAddress), [global, {return, binary}]), maps:put(K, V3, AccIn); InFun(K, V, AccIn) -> maps:put(K, V, AccIn) diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 50e8c9a7d..8fa1e94c3 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -71,8 +71,9 @@ replvar(Cmd, Client = #{username := Username}) -> replvar(Cmd, _) -> Cmd. -repl(S, _Var, undefined) -> +repl(S, _VarPH, undefined) -> S; -repl(S, Var, Val) -> - NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]), - re:replace(S, Var, NVal, [{return, list}]). +repl(S, VarPH, Val) -> + NVal = re:replace(Val, "&", "\\\\&", [global, {return, list}]), + NVarPH = emqx_authz:ph_to_re(VarPH), + re:replace(S, NVarPH, NVal, [{return, list}]). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 884f6e82b..4f7788849 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -32,10 +32,15 @@ -export([ namespace/0 , roots/0 , fields/1 + , validations/0 ]). -import(emqx_schema, [mk_duration/2]). +%%-------------------------------------------------------------------- +%% Hocon Schema +%%-------------------------------------------------------------------- + namespace() -> authz. %% @doc authorization schema is not exported @@ -98,92 +103,24 @@ and the new rules will override all rules from the old config file. }} ]; fields(http_get) -> - [ {type, #{type => http}} - , {enable, #{type => boolean(), - default => true}} - , {url, #{type => url()}} - , {method, #{type => get, default => get }} - , {headers, #{type => map(), - default => #{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> - }, - converter => fun (Headers0) -> - Headers1 = maps:fold(fun(K0, V, AccIn) -> - K1 = iolist_to_binary(string:to_lower(to_list(K0))), - maps:put(K1, V, AccIn) - end, #{}, Headers0), - maps:merge(#{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"keep-alive">> => <<"timeout=5">> - }, Headers1) - end - } - } - , {request_timeout, mk_duration("request timeout", #{default => "30s"})} - ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); + [ {method, #{type => get, default => post}} + , {headers, fun headers_no_content_type/1} + ] ++ http_common_fields(); fields(http_post) -> - [ {type, #{type => http}} - , {enable, #{type => boolean(), - default => true}} - , {url, #{type => url()}} - , {method, #{type => post, - default => get}} - , {headers, #{type => map(), - default => #{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"content-type">> => <<"application/json">> - , <<"keep-alive">> => <<"timeout=5">> - }, - converter => fun (Headers0) -> - Headers1 = maps:fold(fun(K0, V, AccIn) -> - K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))), - maps:put(K1, V, AccIn) - end, #{}, Headers0), - maps:merge(#{ <<"accept">> => <<"application/json">> - , <<"cache-control">> => <<"no-cache">> - , <<"connection">> => <<"keep-alive">> - , <<"content-type">> => <<"application/json">> - , <<"keep-alive">> => <<"timeout=5">> - }, Headers1) - end - } - } - , {request_timeout, mk_duration("request timeout", #{default => "30s"})} - , {body, #{type => map(), - nullable => true - } - } - ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); + [ {method, #{type => post, default => post}} + , {headers, fun headers/1} + ] ++ http_common_fields(); fields(mnesia) -> [ {type, #{type => 'built-in-database'}} , {enable, #{type => boolean(), default => true}} ]; fields(mongo_single) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(single); + mongo_common_fields() ++ emqx_connector_mongo:fields(single); fields(mongo_rs) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(rs); + mongo_common_fields() ++ emqx_connector_mongo:fields(rs); fields(mongo_sharded) -> - [ {collection, #{type => atom()}} - , {selector, #{type => map()}} - , {type, #{type => mongodb}} - , {enable, #{type => boolean(), - default => true}} - ] ++ emqx_connector_mongo:fields(sharded); + mongo_common_fields() ++ emqx_connector_mongo:fields(sharded); fields(mysql) -> connector_fields(mysql) ++ [ {query, query()} ]; @@ -203,10 +140,87 @@ fields(redis_cluster) -> connector_fields(redis, cluster) ++ [ {cmd, query()} ]. +http_common_fields() -> + [ {type, #{type => http}} + , {enable, #{type => boolean(), default => true}} + , {url, #{type => url()}} + , {request_timeout, mk_duration("request timeout", #{default => "30s"})} + , {body, #{type => map(), nullable => true}} + ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)). + +mongo_common_fields() -> + [ {collection, #{type => atom()}} + , {selector, #{type => map()}} + , {type, #{type => mongodb}} + , {enable, #{type => boolean(), + default => true}} + ]. + +validations() -> + [ {check_ssl_opts, fun check_ssl_opts/1} + , {check_headers, fun check_headers/1} + ]. + +headers(type) -> map(); +headers(converter) -> + fun(Headers) -> + maps:merge(default_headers(), transform_header_name(Headers)) + end; +headers(default) -> default_headers(); +headers(_) -> undefined. + +headers_no_content_type(type) -> map(); +headers_no_content_type(converter) -> + fun(Headers) -> + maps:merge(default_headers_no_content_type(), transform_header_name(Headers)) + end; +headers_no_content_type(default) -> default_headers_no_content_type(); +headers_no_content_type(_) -> undefined. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +default_headers() -> + maps:put(<<"content-type">>, + <<"application/json">>, + default_headers_no_content_type()). + +default_headers_no_content_type() -> + #{ <<"accept">> => <<"application/json">> + , <<"cache-control">> => <<"no-cache">> + , <<"connection">> => <<"keep-alive">> + , <<"keep-alive">> => <<"timeout=5">> + }. + +transform_header_name(Headers) -> + maps:fold(fun(K0, V, Acc) -> + K = list_to_binary(string:to_lower(to_list(K0))), + maps:put(K, V, Acc) + end, #{}, Headers). + +check_ssl_opts(Conf) + when Conf =:= #{} -> + true; +check_ssl_opts(Conf) -> + case emqx_authz_http:parse_url(hocon_schema:get_value("config.url", Conf)) of + #{scheme := https} -> + case hocon_schema:get_value("config.ssl.enable", Conf) of + true -> ok; + false -> false + end; + #{scheme := http} -> + ok + end. + +check_headers(Conf) + when Conf =:= #{} -> + true; +check_headers(Conf) -> + Method = to_bin(hocon_schema:get_value("config.method", Conf)), + Headers = hocon_schema:get_value("config.headers", Conf), + Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). + union_array(Item) when is_list(Item) -> hoconsc:array(hoconsc:union(Item)). @@ -229,15 +243,22 @@ connector_fields(DB, Fields) -> catch error:badarg -> list_to_atom(Mod0); - Error -> - erlang:error(Error) + error:Reason -> + erlang:error(Reason) end, [ {type, #{type => DB}} , {enable, #{type => boolean(), default => true}} - ] ++ Mod:fields(Fields). + ] ++ erlang:apply(Mod, fields, [Fields]). to_list(A) when is_atom(A) -> atom_to_list(A); to_list(B) when is_binary(B) -> binary_to_list(B). + +to_bin(A) when is_atom(A) -> + atom_to_binary(A); +to_bin(B) when is_binary(B) -> + B; +to_bin(L) when is_list(L) -> + list_to_binary(L). diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 130e266fb..d965affee 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -36,7 +36,8 @@ init_per_suite(Config) -> meck:expect(emqx_resource, remove, fun(_) -> ok end ), ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], fun set_special_configs/1), + [emqx_connector, emqx_conf, emqx_authz], + fun set_special_configs/1), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 49056ed68..2bca1793d 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -31,7 +31,7 @@ groups() -> init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps( - [emqx_conf, emqx_authz], + [emqx_connector, emqx_conf, emqx_authz], fun set_special_configs/1 ), Config. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 417fa49f7..5b2b62d82 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -17,220 +17,266 @@ -behaviour(minirest_api). --export([api_spec/0]). +-include_lib("typerefl/include/types.hrl"). --export([ list_create_bridges_in_cluster/2 - , list_local_bridges/1 - , crud_bridges_in_cluster/2 - , manage_bridges/2 +-import(hoconsc, [mk/2, array/1, enum/1]). + +%% Swagger specs from hocon schema +-export([api_spec/0, paths/0, schema/1, namespace/0]). + +%% API callbacks +-export(['/bridges'/2, '/bridges/:id'/2, + '/nodes/:node/bridges/:id/operation/:operation'/2]). + +-export([ list_local_bridges/1 , lookup_from_local_node/2 ]). -define(TYPES, [mqtt, http]). + +-define(CONN_TYPES, [mqtt]). + -define(TRY_PARSE_ID(ID, EXPR), try emqx_bridge:parse_bridge_id(Id) of {BridgeType, BridgeName} -> EXPR catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge Ids must be of format {type}:{name}">>}} end). -define(METRICS(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{ matched => MATCH, success => SUCC, failed => FAILED, - speed => RATE, - speed_last5m => RATE_5, - speed_max => RATE_MAX + rate => RATE, + rate_last5m => RATE_5, + rate_max => RATE_MAX }). -define(metrics(MATCH, SUCC, FAILED, RATE, RATE_5, RATE_MAX), #{ matched := MATCH, success := SUCC, failed := FAILED, - speed := RATE, - speed_last5m := RATE_5, - speed_max := RATE_MAX + rate := RATE, + rate_last5m := RATE_5, + rate_max := RATE_MAX }). -req_schema() -> - Schema = [ - case maps:to_list(emqx:get_raw_config([bridges, T], #{})) of - %% the bridge is not configured, so we have no method to get the schema - [] -> #{}; - [{_K, Conf} | _] -> - emqx_mgmt_api_configs:gen_schema(Conf) - end - || T <- ?TYPES], - #{'oneOf' => Schema}. - -node_schema() -> - #{type => string, example => "emqx@127.0.0.1"}. - -status_schema() -> - #{type => string, enum => [connected, disconnected]}. - -metrics_schema() -> - #{ type => object - , properties => #{ - matched => #{type => integer, example => "0"}, - success => #{type => integer, example => "0"}, - failed => #{type => integer, example => "0"}, - speed => #{type => number, format => float, example => "0.0"}, - speed_last5m => #{type => number, format => float, example => "0.0"}, - speed_max => #{type => number, format => float, example => "0.0"} - } - }. - -per_node_schema(Key, Schema) -> - #{ - type => array, - items => #{ - type => object, - properties => #{ - node => node_schema(), - Key => Schema - } - } - }. - -resp_schema() -> - AddMetadata = fun(Prop) -> - Prop#{status => status_schema(), - node_status => per_node_schema(status, status_schema()), - metrics => metrics_schema(), - node_metrics => per_node_schema(metrics, metrics_schema()), - id => #{type => string, example => "http:my_http_bridge"}, - bridge_type => #{type => string, enum => ?TYPES}, - node => node_schema() - } - end, - more_props_resp_schema(AddMetadata). - -more_props_resp_schema(AddMetadata) -> - #{'oneOf' := Schema} = req_schema(), - Schema1 = [S#{properties => AddMetadata(Prop)} - || S = #{properties := Prop} <- Schema], - #{'oneOf' => Schema1}. +namespace() -> "bridge". api_spec() -> - {bridge_apis(), []}. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). -bridge_apis() -> - [list_all_bridges_api(), crud_bridges_apis(), operation_apis()]. +paths() -> ["/bridges", "/bridges/:id", "/nodes/:node/bridges/:id/operation/:operation"]. -list_all_bridges_api() -> - ReqSchema = more_props_resp_schema(fun(Prop) -> - Prop#{id => #{type => string, required => true}} - end), - RespSchema = resp_schema(), - Metadata = #{ +error_schema(Code, Message) -> + [ {code, mk(string(), #{example => Code})} + , {message, mk(string(), #{example => Message})} + ]. + +get_response_body_schema() -> + emqx_dashboard_swagger:schema_with_examples(emqx_bridge_schema:get_response(), + bridge_info_examples(get)). + +param_path_node() -> + path_param(node, binary(), atom_to_binary(node(), utf8)). + +param_path_operation() -> + path_param(operation, enum([start, stop, restart]), <<"start">>). + +param_path_id() -> + path_param(id, binary(), <<"http:my_http_bridge">>). + +path_param(Name, Type, Example) -> + {Name, mk(Type, + #{ in => path + , required => true + , example => Example + })}. + +bridge_info_array_example(Method) -> + [Config || #{value := Config} <- maps:values(bridge_info_examples(Method))]. + +bridge_info_examples(Method) -> + maps:merge(conn_bridge_examples(Method), #{ + <<"http_bridge">> => #{ + summary => <<"HTTP Bridge">>, + value => info_example(http, awesome, Method) + } + }). + +conn_bridge_examples(Method) -> + lists:foldl(fun(Type, Acc) -> + SType = atom_to_list(Type), + KeyIngress = bin(SType ++ "_ingress"), + KeyEgress = bin(SType ++ "_egress"), + maps:merge(Acc, #{ + KeyIngress => #{ + summary => bin(string:uppercase(SType) ++ " Ingress Bridge"), + value => info_example(Type, ingress, Method) + }, + KeyEgress => #{ + summary => bin(string:uppercase(SType) ++ " Egress Bridge"), + value => info_example(Type, egress, Method) + } + }) + end, #{}, ?CONN_TYPES). + +info_example(Type, Direction, Method) -> + maps:merge(info_example_basic(Type, Direction), + method_example(Type, Direction, Method)). + +method_example(Type, Direction, get) -> + SType = atom_to_list(Type), + SDir = atom_to_list(Direction), + SName = "my_" ++ SDir ++ "_" ++ SType ++ "_bridge", + #{ + id => bin(SType ++ ":" ++ SName), + type => bin(SType), + name => bin(SName) + }; +method_example(Type, Direction, post) -> + SType = atom_to_list(Type), + SDir = atom_to_list(Direction), + SName = "my_" ++ SDir ++ "_" ++ SType ++ "_bridge", + #{ + type => bin(SType), + name => bin(SName) + }; +method_example(_Type, _Direction, put) -> + #{}. + +info_example_basic(http, _) -> + #{ + url => <<"http://localhost:9901/messages/${topic}">>, + request_timeout => <<"30s">>, + connect_timeout => <<"30s">>, + max_retries => 3, + retry_interval => <<"10s">>, + pool_type => <<"random">>, + pool_size => 4, + enable_pipelining => true, + ssl => #{enable => false}, + from_local_topic => <<"emqx_http/#">>, + method => post, + body => <<"${payload}">> + }; +info_example_basic(mqtt, ingress) -> + #{ + connector => <<"mqtt:my_mqtt_connector">>, + direction => ingress, + from_remote_topic => <<"aws/#">>, + subscribe_qos => 1, + to_local_topic => <<"from_aws/${topic}">>, + payload => <<"${payload}">>, + qos => <<"${qos}">>, + retain => <<"${retain}">> + }; +info_example_basic(mqtt, egress) -> + #{ + connector => <<"mqtt:my_mqtt_connector">>, + direction => egress, + from_local_topic => <<"emqx/#">>, + to_remote_topic => <<"from_emqx/${topic}">>, + payload => <<"${payload}">>, + qos => 1, + retain => false + }. + +schema("/bridges") -> + #{ + operationId => '/bridges', get => #{ + tags => [<<"bridges">>], + summary => <<"List Bridges">>, description => <<"List all created bridges">>, responses => #{ - <<"200">> => emqx_mgmt_util:array_schema(resp_schema(), - <<"A list of the bridges">>) + 200 => emqx_dashboard_swagger:schema_with_example( + array(emqx_bridge_schema:get_response()), + bridge_info_array_example(get)) } }, post => #{ + tags => [<<"bridges">>], + summary => <<"Create Bridge">>, description => <<"Create a new bridge">>, - 'requestBody' => emqx_mgmt_util:schema(ReqSchema), + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_schema:post_request(), + bridge_info_examples(post)), responses => #{ - <<"201">> => emqx_mgmt_util:schema(RespSchema, <<"Bridge created">>), - <<"400">> => emqx_mgmt_util:error_schema(<<"Create bridge failed">>, - ['UPDATE_FAILED']) + 201 => get_response_body_schema(), + 400 => error_schema('BAD_ARG', "Create bridge failed") } } - }, - {"/bridges/", Metadata, list_create_bridges_in_cluster}. + }; -crud_bridges_apis() -> - ReqSchema = req_schema(), - RespSchema = resp_schema(), - Metadata = #{ +schema("/bridges/:id") -> + #{ + operationId => '/bridges/:id', get => #{ + tags => [<<"bridges">>], + summary => <<"Get Bridge">>, description => <<"Get a bridge by Id">>, parameters => [param_path_id()], responses => #{ - <<"200">> => emqx_mgmt_util:array_schema(RespSchema, - <<"The details of the bridge">>), - <<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND']) + 200 => get_response_body_schema(), + 404 => error_schema('NOT_FOUND', "Bridge not found") } }, put => #{ + tags => [<<"bridges">>], + summary => <<"Update Bridge">>, description => <<"Update a bridge">>, parameters => [param_path_id()], - 'requestBody' => emqx_mgmt_util:schema(ReqSchema), + requestBody => emqx_dashboard_swagger:schema_with_examples( + emqx_bridge_schema:put_request(), + bridge_info_examples(put)), responses => #{ - <<"200">> => emqx_mgmt_util:array_schema(RespSchema, <<"Bridge updated">>), - <<"400">> => emqx_mgmt_util:error_schema(<<"Update bridge failed">>, - ['UPDATE_FAILED']) + 200 => get_response_body_schema(), + 400 => error_schema('BAD_ARG', "Update bridge failed") } }, delete => #{ + tags => [<<"bridges">>], + summary => <<"Delete Bridge">>, description => <<"Delete a bridge">>, parameters => [param_path_id()], responses => #{ - <<"204">> => emqx_mgmt_util:schema(<<"Bridge deleted">>), - <<"404">> => emqx_mgmt_util:error_schema(<<"Bridge not found">>, ['NOT_FOUND']) + 204 => <<"Bridge deleted">> } } - }, - {"/bridges/:id", Metadata, crud_bridges_in_cluster}. + }; -operation_apis() -> - Metadata = #{ +schema("/nodes/:node/bridges/:id/operation/:operation") -> + #{ + operationId => '/nodes/:node/bridges/:id/operation/:operation', post => #{ + tags => [<<"bridges">>], + summary => <<"Start/Stop/Restart Bridge">>, description => <<"Start/Stop/Restart bridges on a specific node">>, parameters => [ param_path_node(), param_path_id(), - param_path_operation()], + param_path_operation() + ], responses => #{ - <<"500">> => emqx_mgmt_util:error_schema(<<"Operation Failed">>, - ['INTERNAL_ERROR']), - <<"200">> => emqx_mgmt_util:schema(<<"Operation success">>)}}}, - {"/nodes/:node/bridges/:id/operation/:operation", Metadata, manage_bridges}. - -param_path_node() -> - #{ - name => node, - in => path, - schema => #{type => string}, - required => true, - example => node() + 500 => error_schema('INTERNAL_ERROR', "Operation Failed"), + 200 => <<"Operation success">> + } + } }. -param_path_id() -> - #{ - name => id, - in => path, - schema => #{type => string}, - required => true - }. - -param_path_operation()-> - #{ - name => operation, - in => path, - required => true, - schema => #{ - type => string, - enum => [start, stop, restart]}, - example => restart - }. - -list_create_bridges_in_cluster(post, #{body := #{<<"id">> := Id} = Conf}) -> - ?TRY_PARSE_ID(Id, - case emqx_bridge:lookup(BridgeType, BridgeName) of - {ok, _} -> {400, #{code => 'ALREADY_EXISTS', message => <<"bridge already exists">>}}; - {error, not_found} -> - case ensure_bridge(BridgeType, BridgeName, maps:remove(<<"id">>, Conf)) of - ok -> lookup_from_all_nodes(Id, BridgeType, BridgeName, 201); - {error, Error} -> {400, Error} - end - end); -list_create_bridges_in_cluster(get, _Params) -> +'/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf}) -> + BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()), + case emqx_bridge:lookup(BridgeType, BridgeName) of + {ok, _} -> {400, #{code => 'ALREADY_EXISTS', message => <<"bridge already exists">>}}; + {error, not_found} -> + case ensure_bridge_created(BridgeType, BridgeName, Conf) of + ok -> lookup_from_all_nodes(BridgeType, BridgeName, 201); + {error, Error} -> {400, Error} + end + end; +'/bridges'(get, _Params) -> {200, zip_bridges([list_local_bridges(Node) || Node <- mria_mnesia:running_nodes()])}. list_local_bridges(Node) when Node =:= node() -> @@ -238,22 +284,22 @@ list_local_bridges(Node) when Node =:= node() -> list_local_bridges(Node) -> rpc_call(Node, list_local_bridges, [Node]). -crud_bridges_in_cluster(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes(Id, BridgeType, BridgeName, 200)); +'/bridges/:id'(get, #{bindings := #{id := Id}}) -> + ?TRY_PARSE_ID(Id, lookup_from_all_nodes(BridgeType, BridgeName, 200)); -crud_bridges_in_cluster(put, #{bindings := #{id := Id}, body := Conf}) -> +'/bridges/:id'(put, #{bindings := #{id := Id}, body := Conf}) -> ?TRY_PARSE_ID(Id, case emqx_bridge:lookup(BridgeType, BridgeName) of {ok, _} -> - case ensure_bridge(BridgeType, BridgeName, Conf) of - ok -> lookup_from_all_nodes(Id, BridgeType, BridgeName, 200); + case ensure_bridge_created(BridgeType, BridgeName, Conf) of + ok -> lookup_from_all_nodes(BridgeType, BridgeName, 200); {error, Error} -> {400, Error} end; {error, not_found} -> {404, #{code => 'NOT_FOUND', message => <<"bridge not found">>}} end); -crud_bridges_in_cluster(delete, #{bindings := #{id := Id}}) -> +'/bridges/:id'(delete, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID(Id, case emqx_conf:remove(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], #{override_to => cluster}) of @@ -262,12 +308,12 @@ crud_bridges_in_cluster(delete, #{bindings := #{id := Id}}) -> {500, #{code => 102, message => emqx_resource_api:stringify(Reason)}} end). -lookup_from_all_nodes(Id, BridgeType, BridgeName, SuccCode) -> +lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> case rpc_multicall(lookup_from_local_node, [BridgeType, BridgeName]) of {ok, [{ok, _} | _] = Results} -> {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> - {404, error_msg('NOT_FOUND', <<"not_found: ", Id/binary>>)}; + {404, error_msg('NOT_FOUND', <<"not_found">>)}; {error, ErrL} -> {500, error_msg('UNKNOWN_ERROR', ErrL)} end. @@ -278,7 +324,8 @@ lookup_from_local_node(BridgeType, BridgeName) -> Error -> Error end. -manage_bridges(post, #{bindings := #{node := Node, id := Id, operation := Op}}) -> +'/nodes/:node/bridges/:id/operation/:operation'(post, #{bindings := + #{node := Node, id := Id, operation := Op}}) -> OperFun = fun (<<"start">>) -> start; (<<"stop">>) -> stop; @@ -292,9 +339,10 @@ manage_bridges(post, #{bindings := #{node := Node, id := Id, operation := Op}}) {500, #{code => 102, message => emqx_resource_api:stringify(Reason)}} end). -ensure_bridge(BridgeType, BridgeName, Conf) -> - case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], Conf, - #{override_to => cluster}) of +ensure_bridge_created(BridgeType, BridgeName, Conf) -> + Conf1 = maps:without([<<"type">>, <<"name">>], Conf), + case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], + Conf1, #{override_to => cluster}) of {ok, _} -> ok; {error, Reason} -> {error, error_msg('BAD_ARG', Reason)} @@ -346,12 +394,14 @@ aggregate_metrics(AllMetrics) -> end, InitMetrics, AllMetrics). format_resp(#{id := Id, raw_config := RawConf, - resource_data := #{mod := Mod, status := Status, metrics := Metrics}}) -> + resource_data := #{status := Status, metrics := Metrics}}) -> + {Type, Name} = emqx_bridge:parse_bridge_id(Id), IsConnected = fun(started) -> connected; (_) -> disconnected end, RawConf#{ id => Id, + type => Type, + name => Name, node => node(), - bridge_type => emqx_bridge:bridge_type(Mod), status => IsConnected(Status), metrics => Metrics }. @@ -378,4 +428,7 @@ rpc_call(Node, Mod, Fun, Args) -> error_msg(Code, Msg) when is_binary(Msg) -> #{code => Code, message => Msg}; error_msg(Code, Msg) -> - #{code => Code, message => list_to_binary(io_lib:format("~p", [Msg]))}. + #{code => Code, message => bin(io_lib:format("~p", [Msg]))}. + +bin(S) when is_list(S) -> + list_to_binary(S). diff --git a/apps/emqx_bridge/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl new file mode 100644 index 000000000..2bef474bd --- /dev/null +++ b/apps/emqx_bridge/src/emqx_bridge_http_schema.erl @@ -0,0 +1,95 @@ +-module(emqx_bridge_http_schema). + +-include_lib("typerefl/include/types.hrl"). + +-import(hoconsc, [mk/2, enum/1]). + +-export([roots/0, fields/1]). + +%%====================================================================================== +%% Hocon Schema Definitions +roots() -> []. + +fields("bridge") -> + basic_config() ++ + [ {url, mk(binary(), + #{ nullable => false + , desc =>""" +The URL of the HTTP Bridge.
+Template with variables is allowed in the path, but variables cannot be used in the scheme, host, +or port part.
+For example, http://localhost:9901/${topic} is allowed, but + http://${host}:9901/message or http://localhost:${port}/message +is not allowed. +""" + })} + , {from_local_topic, mk(binary(), + #{ desc =>""" +The MQTT topic filter to be forwarded to the HTTP server. All MQTT PUBLISH messages which topic +match the from_local_topic will be forwarded.
+NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also from_local_topic is configured, then both the data got from the rule and the MQTT messages that matches +from_local_topic will be forwarded. +""" + })} + , {method, mk(method(), + #{ default => post + , desc =>""" +The method of the HTTP request. All the available methods are: post, put, get, delete.
+Template with variables is allowed.
+""" + })} + , {headers, mk(map(), + #{ default => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keep-alive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">>} + , desc =>""" +The headers of the HTTP request.
+Template with variables is allowed. +""" + }) + } + , {body, mk(binary(), + #{ default => <<"${payload}">> + , desc =>""" +The body of the HTTP request.
+Template with variables is allowed. +""" + })} + , {request_timeout, mk(emqx_schema:duration_ms(), + #{ default => <<"30s">> + , desc =>""" +How long will the HTTP request timeout. +""" + })} + ]; + +fields("post") -> + [ type_field() + , name_field() + ] ++ fields("bridge"); + +fields("put") -> + fields("bridge"); + +fields("get") -> + [ id_field() + ] ++ fields("post"). + +basic_config() -> + proplists:delete(base_url, emqx_connector_http:fields(config)). + +%%====================================================================================== +id_field() -> + {id, mk(binary(), #{desc => "The Bridge Id", example => "http:my_http_bridge"})}. + +type_field() -> + {type, mk(http, #{desc => "The Bridge Type"})}. + +name_field() -> + {name, mk(binary(), #{desc => "The Bridge Name"})}. + +method() -> + enum([post, put, get, delete]). diff --git a/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl new file mode 100644 index 000000000..d2cf6b1a8 --- /dev/null +++ b/apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl @@ -0,0 +1,62 @@ +-module(emqx_bridge_mqtt_schema). + +-include_lib("typerefl/include/types.hrl"). + +-import(hoconsc, [mk/2]). + +-export([roots/0, fields/1]). + +%%====================================================================================== +%% Hocon Schema Definitions +roots() -> []. + +fields("ingress") -> + [ direction(ingress, emqx_connector_mqtt_schema:ingress_desc()) + , emqx_bridge_schema:connector_name() + ] ++ proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress")); + +fields("egress") -> + [ direction(egress, emqx_connector_mqtt_schema:egress_desc()) + , emqx_bridge_schema:connector_name() + ] ++ emqx_connector_mqtt_schema:fields("egress"); + +fields("post_ingress") -> + [ type_field() + , name_field() + ] ++ fields("ingress"); +fields("post_egress") -> + [ type_field() + , name_field() + ] ++ fields("egress"); + +fields("put_ingress") -> + fields("ingress"); +fields("put_egress") -> + fields("egress"); + +fields("get_ingress") -> + [ id_field() + ] ++ fields("post_ingress"); +fields("get_egress") -> + [ id_field() + ] ++ fields("post_egress"). + +%%====================================================================================== +direction(Dir, Desc) -> + {direction, mk(Dir, + #{ nullable => false + , desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.
" + ++ Desc + })}. + +id_field() -> + {id, mk(binary(), #{desc => "The Bridge Id", example => "mqtt:my_mqtt_bridge"})}. + +type_field() -> + {type, mk(mqtt, #{desc => "The Bridge Type"})}. + +name_field() -> + {name, mk(binary(), + #{ desc => "The Bridge Name" + , example => "some_bridge_name" + })}. diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl index 26a1d5bd1..ec875d0a4 100644 --- a/apps/emqx_bridge/src/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -2,122 +2,63 @@ -include_lib("typerefl/include/types.hrl"). +-import(hoconsc, [mk/2, ref/2]). + -export([roots/0, fields/1]). +-export([ get_response/0 + , put_request/0 + , post_request/0 + ]). + +-export([ connector_name/0 + ]). + %%====================================================================================== %% Hocon Schema Definitions -roots() -> [bridges]. +-define(CONN_TYPES, [mqtt]). -fields(bridges) -> - [ {mqtt, - sc(hoconsc:map(name, hoconsc:union([ ref("ingress_mqtt_bridge") - , ref("egress_mqtt_bridge") - ])), - #{ desc => "MQTT bridges" - })} - , {http, - sc(hoconsc:map(name, ref("http_bridge")), - #{ desc => "HTTP bridges" - })} - ]; - -fields("ingress_mqtt_bridge") -> - [ direction(ingress, emqx_connector_mqtt_schema:ingress_desc()) - , connector_name() - ] ++ proplists:delete(hookpoint, emqx_connector_mqtt_schema:fields("ingress")); - -fields("egress_mqtt_bridge") -> - [ direction(egress, emqx_connector_mqtt_schema:egress_desc()) - , connector_name() - ] ++ emqx_connector_mqtt_schema:fields("egress"); - -fields("http_bridge") -> - basic_config_http() ++ - [ {url, - sc(binary(), - #{ nullable => false - , desc =>""" -The URL of the HTTP Bridge.
-Template with variables is allowed in the path, but variables cannot be used in the scheme, host, -or port part.
-For example, http://localhost:9901/${topic} is allowed, but - http://${host}:9901/message or http://localhost:${port}/message -is not allowed. -""" - })} - , {from_local_topic, - sc(binary(), - #{ desc =>""" -The MQTT topic filter to be forwarded to the HTTP server. All MQTT PUBLISH messages which topic -match the from_local_topic will be forwarded.
-NOTE: if this bridge is used as the output of a rule (emqx rule engine), and also from_local_topic is configured, then both the data got from the rule and the MQTT messages that matches -from_local_topic will be forwarded. -""" - })} - , {method, - sc(method(), - #{ default => post - , desc =>""" -The method of the HTTP request. All the available methods are: post, put, get, delete.
-Template with variables is allowed.
-""" - })} - , {headers, - sc(map(), - #{ default => #{ - <<"accept">> => <<"application/json">>, - <<"cache-control">> => <<"no-cache">>, - <<"connection">> => <<"keep-alive">>, - <<"content-type">> => <<"application/json">>, - <<"keep-alive">> => <<"timeout=5">>} - , desc =>""" -The headers of the HTTP request.
-Template with variables is allowed. -""" - }) - } - , {body, - sc(binary(), - #{ default => <<"${payload}">> - , desc =>""" -The body of the HTTP request.
-Template with variables is allowed. -""" - })} - , {request_timeout, - sc(emqx_schema:duration_ms(), - #{ default => <<"30s">> - , desc =>""" -How long will the HTTP request timeout. -""" - })} - ]. - -direction(Dir, Desc) -> - {direction, - sc(Dir, - #{ nullable => false - , desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.
" ++ - Desc - })}. +%%====================================================================================== +%% For HTTP APIs +get_response() -> + http_schema("get"). connector_name() -> {connector, - sc(binary(), + mk(binary(), #{ nullable => false , desc =>""" The connector name to be used for this bridge. -Connectors are configured by 'connectors.. +Connectors are configured as 'connectors.{type}.{name}', +for example 'connectors.http.mybridge'. """ })}. -basic_config_http() -> - proplists:delete(base_url, emqx_connector_http:fields(config)). +put_request() -> + http_schema("put"). -method() -> - hoconsc:enum([post, put, get, delete]). +post_request() -> + http_schema("post"). -sc(Type, Meta) -> hoconsc:mk(Type, Meta). +http_schema(Method) -> + Schemas = lists:flatmap(fun(Type) -> + [ref(schema_mod(Type), Method ++ "_ingress"), + ref(schema_mod(Type), Method ++ "_egress")] + end, ?CONN_TYPES), + hoconsc:union([ref(emqx_bridge_http_schema, Method) + | Schemas]). -ref(Field) -> hoconsc:ref(?MODULE, Field). +%%====================================================================================== +%% For config files +roots() -> [bridges]. + +fields(bridges) -> + [{http, mk(hoconsc:map(name, ref(emqx_bridge_http_schema, "bridge")), #{})}] + ++ [{T, mk(hoconsc:map(name, hoconsc:union([ + ref(schema_mod(T), "ingress"), + ref(schema_mod(T), "egress") + ])), #{})} || T <- ?CONN_TYPES]. + +schema_mod(Type) -> + list_to_atom(lists:concat(["emqx_bridge_", Type, "_schema"])). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 23d4691f5..52c8a32de 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -21,7 +21,9 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"bridges: {}">>). --define(TEST_ID, <<"http:test_bridge">>). +-define(BRIDGE_TYPE, <<"http">>). +-define(BRIDGE_NAME, <<"test_bridge">>). +-define(BRIDGE_ID, <<"http:test_bridge">>). -define(URL(PORT, PATH), list_to_binary( io_lib:format("http://localhost:~s/~s", [integer_to_list(PORT), PATH]))). @@ -134,11 +136,15 @@ t_http_crud_apis(_) -> %% POST /bridges/ will create a bridge URL1 = ?URL(Port, "path1"), {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}), + ?HTTP_BRIDGE(URL1)#{ + <<"type">> => ?BRIDGE_TYPE, + <<"name">> => ?BRIDGE_NAME + }), %ct:pal("---bridge: ~p", [Bridge]), - ?assertMatch(#{ <<"id">> := ?TEST_ID - , <<"bridge_type">> := <<"http">> + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ , <<"node_status">> := [_|_] , <<"metrics">> := _ @@ -148,7 +154,10 @@ t_http_crud_apis(_) -> %% create a again returns an error {ok, 400, RetMsg} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}), + ?HTTP_BRIDGE(URL1)#{ + <<"type">> => ?BRIDGE_TYPE, + <<"name">> => ?BRIDGE_NAME + }), ?assertMatch( #{ <<"code">> := _ , <<"message">> := <<"bridge already exists">> @@ -156,10 +165,11 @@ t_http_crud_apis(_) -> %% update the request-path of the bridge URL2 = ?URL(Port, "path2"), - {ok, 200, Bridge2} = request(put, uri(["bridges", ?TEST_ID]), + {ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]), ?HTTP_BRIDGE(URL2)), - ?assertMatch(#{ <<"id">> := ?TEST_ID - , <<"bridge_type">> := <<"http">> + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ , <<"node_status">> := [_|_] , <<"metrics">> := _ @@ -169,8 +179,9 @@ t_http_crud_apis(_) -> %% list all bridges again, assert Bridge2 is in it {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []), - ?assertMatch([#{ <<"id">> := ?TEST_ID - , <<"bridge_type">> := <<"http">> + ?assertMatch([#{ <<"id">> := ?BRIDGE_ID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ , <<"node_status">> := [_|_] , <<"metrics">> := _ @@ -179,9 +190,10 @@ t_http_crud_apis(_) -> }], jsx:decode(Bridge2Str)), %% get the bridge by id - {ok, 200, Bridge3Str} = request(get, uri(["bridges", ?TEST_ID]), []), - ?assertMatch(#{ <<"id">> := ?TEST_ID - , <<"bridge_type">> := <<"http">> + {ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []), + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ , <<"node_status">> := [_|_] , <<"metrics">> := _ @@ -190,11 +202,11 @@ t_http_crud_apis(_) -> }, jsx:decode(Bridge3Str)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?TEST_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []), %% update a deleted bridge returns an error - {ok, 404, ErrMsg2} = request(put, uri(["bridges", ?TEST_ID]), + {ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]), ?HTTP_BRIDGE(URL2)), ?assertMatch( #{ <<"code">> := _ @@ -206,11 +218,15 @@ t_start_stop_bridges(_) -> Port = start_http_server(fun handle_fun_200_ok/1), URL1 = ?URL(Port, "abc"), {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?HTTP_BRIDGE(URL1)#{<<"id">> => ?TEST_ID}), + ?HTTP_BRIDGE(URL1)#{ + <<"type">> => ?BRIDGE_TYPE, + <<"name">> => ?BRIDGE_NAME + }), %ct:pal("the bridge ==== ~p", [Bridge]), ?assertMatch( - #{ <<"id">> := ?TEST_ID - , <<"bridge_type">> := <<"http">> + #{ <<"id">> := ?BRIDGE_ID + , <<"type">> := ?BRIDGE_TYPE + , <<"name">> := ?BRIDGE_NAME , <<"status">> := _ , <<"node_status">> := [_|_] , <<"metrics">> := _ @@ -219,42 +235,42 @@ t_start_stop_bridges(_) -> }, jsx:decode(Bridge)), %% stop it {ok, 200, <<>>} = request(post, - uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "stop"]), + uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "stop"]), <<"">>), - {ok, 200, Bridge2} = request(get, uri(["bridges", ?TEST_ID]), []), - ?assertMatch(#{ <<"id">> := ?TEST_ID + {ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []), + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID , <<"status">> := <<"disconnected">> }, jsx:decode(Bridge2)), %% start again {ok, 200, <<>>} = request(post, - uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "start"]), + uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "start"]), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", ?TEST_ID]), []), - ?assertMatch(#{ <<"id">> := ?TEST_ID + {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID , <<"status">> := <<"connected">> }, jsx:decode(Bridge3)), %% restart an already started bridge {ok, 200, <<>>} = request(post, - uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "restart"]), + uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "restart"]), <<"">>), - {ok, 200, Bridge3} = request(get, uri(["bridges", ?TEST_ID]), []), - ?assertMatch(#{ <<"id">> := ?TEST_ID + {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []), + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID , <<"status">> := <<"connected">> }, jsx:decode(Bridge3)), %% stop it again {ok, 200, <<>>} = request(post, - uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "stop"]), + uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "stop"]), <<"">>), %% restart a stopped bridge {ok, 200, <<>>} = request(post, - uri(["nodes", node(), "bridges", ?TEST_ID, "operation", "restart"]), + uri(["nodes", node(), "bridges", ?BRIDGE_ID, "operation", "restart"]), <<"">>), - {ok, 200, Bridge4} = request(get, uri(["bridges", ?TEST_ID]), []), - ?assertMatch(#{ <<"id">> := ?TEST_ID + {ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []), + ?assertMatch(#{ <<"id">> := ?BRIDGE_ID , <<"status">> := <<"connected">> }, jsx:decode(Bridge4)), %% delete the bridge - {ok, 204, <<>>} = request(delete, uri(["bridges", ?TEST_ID]), []), + {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []), {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []). %%-------------------------------------------------------------------- diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index e57bc4869..ee869f3a3 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -27,12 +27,27 @@ node { ## Default: "{{ platform_data_dir }}/" data_dir = "{{ platform_data_dir }}/" - ## Dir of crash dump file. + ## Location of crash dump file. ## - ## @doc node.crash_dump_dir - ## ValueType: Folder - ## Default: "{{ platform_log_dir }}/" - crash_dump_dir = "{{ platform_log_dir }}/" + ## @doc node.crash_dump_file + ## ValueType: File + ## Default: "{{ platform_log_dir }}/erl_crash.dump" + crash_dump_file = "{{ platform_log_dir }}/erl_crash.dump" + + ## The number of seconds that the broker is allowed to spend writing + ## a crash dump + ## + ## @doc node.crash_dump_seconds + ## ValueType: seconds + ## Default: 30s + crash_dump_seconds = 30s + + ## The maximum size of a crash dump file in bytes. + ## + ## @doc node.crash_dump_bytes + ## ValueType: bytes + ## Default: 100MB + crash_dump_bytes = 100MB ## Global GC Interval. ## diff --git a/apps/emqx_conf/etc/emqx_conf.md b/apps/emqx_conf/etc/emqx_conf.md new file mode 100644 index 000000000..f68411bff --- /dev/null +++ b/apps/emqx_conf/etc/emqx_conf.md @@ -0,0 +1,192 @@ +EMQ X configuration file is in [HOCON](https://github.com/emqx/hocon) format. +HOCON, or Human-Optimized Config Object Notation is a format for human-readable data, +and a superset of JSON. + +## Syntax + +In config file the values can be notated as JSON like ojbects, such as +``` +node { + name = "emqx@127.0.0.1" + cookie = "mysecret" +} +``` + +Another equivalent representation is flat, suh as + +``` +node.name="127.0.0.1" +node.cookie="mysecret" +``` + +This flat format is almost backward compatible with EMQ X's config file format +in 4.x series (the so called 'cuttlefish' format). + +It is 'almost' compabile because the often HOCON requires strings to be quoted, +while cuttlefish treats all characters to the right of the `=` mark as the value. + +e.g. cuttlefish: `node.name = emqx@127.0.0.1`, HOCON: `node.name = "emqx@127.0.0.1"` + +Strings without special characters in them can be unquoted in HOCON too, +e.g. `foo`, `foo_bar`, `foo_bar_1`: + +For more HOCON syntax, pelase refer to the [specification](https://github.com/lightbend/config/blob/main/HOCON.md) + +## Schema + +To make the HOCON objects type-safe, EMQ X introduded a schema for it. +The schema defines data types, and data fields' names and metadata for config value validation +and more. In fact, this config document itself is generated from schema metadata. + +### Complex Data Types + +There are 4 complex data types in EMQ X's HOCON config: + +1. Struct: Named using an unquoted string, followed by a pre-defined list of fields, + fields can not start with a number, and are only allowed to use + lowercase letters and underscores as word separater. +1. Map: Map is like Struct, however the fields are not pre-defined. + 1-based index number can also be used as map keys for an alternative + representation of an Array. +1. Union: `MemberType1 | MemberType2 | ...` +1. Array: `[ElementType]` + +### Primitive Data Types + +Complex types define data 'boxes' wich may contain other complex data +or primitive values. +There are quite some different primitive types, to name a fiew: + +* `atom()` +* `boolean()` +* `string()` +* `integer()` +* `float()` +* `number()` +* `binary()` # another format of string() +* `emqx_schema:duration()` # time duration, another format of integer() +* ... + +The primitive types are mostly self-describing, some are built-in, such +as `atom()`, some are defiend in EMQ X modules, such as `emqx_schema:duration()`. + +### Config Paths + +If we consider the whole EMQ X config as a tree, +to reference a primitive value, we can use a dot-separated names form string for +the path from the tree-root (always a Struct) down to the primitive values at tree-leaves. + +Each segment of the dotted string is a Struct filed name or Map key. +For Array elements, 1-based index is used. + +below are some examples + +``` +node.name="emqx.127.0.0.1" +zone.zone1.max_packet_size="10M" +authentication.1.enable=true +``` + +### Environment varialbes + +Environment variables can be used to define or override config values. + +Due to the fact that dots (`.`) are not allowed in environment variables, dots are +replaced with double-underscores (`__`). + +And a the `EMQX_` prefix is used as the namespace. + +For example `node.name` can be represented as `EMQX_NODE__NAME` + +Environment varialbe values are parsed as hocon values, this allows users +to even set complex values from environment variables. + +For example, this environment variable sets an array value. + +``` +export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]" +``` + +Unknown environment variables are logged as a `warning` level log, for example: + +``` +[warning] unknown_env_vars: ["EMQX_AUTHENTICATION__ENABLED"] +``` + +because the field name is `enable`, not `enabled`. + +NOTE: Unknown root keys are however silently discarded. + +### Config overlay + +HOCON values are overlayed, earlier defined values are at layers closer to the bottom. +The overall order of the overlay rules from bottom up are: + +1. `emqx.conf` the base config file +1. `EMQX_` prfixed environment variables +1. Cluster override file, the path of which is configured as `cluster_override_conf_file` in the lower layers +1. Local override file, the path of which is configured as `local_override_conf_file` in the lower layers + +Below are the rules of config value overlay. + +#### Struct Fileds + +Later config values overwrites earlier values. +For example, in below config, the last line `debug` overwrites `errro` for +console log handler's `level` config, but leaving `enable` unchanged. +``` +log { + console_handler{ + enable=true, + level=error + } +} + +## ... more configs ... + +log.console_handler.level=debug +``` + +#### Map Values + +Maps are like structs, only the files are user-defined rather than +the config schema. For instance, `zone1` in the exampele below. + +``` +zone { + zone1 { + mqtt.max_packet_size = 1M + } +} + +## The maximum packet size can be defined as above, +## then overriden as below + +zone.zone1.mqtt.max_packet_size = 10M +``` + +#### Array Elements + +Arrays in EMQ X config have two different representations + +* list, such as: `[1, 2, 3]` +* indexed-map, such as: `{"1"=1, "2"=2, "3"=3}` + +Dot-separated paths with number in it are parsed to indexed-maps +e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}` + +Indexed-map arrays can be used to override list arrays: + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +# we can disable this authentication provider with: +authentication.1.enable=false +``` +However, list arrays do not get recursively merged into indexed-map arrays. +e.g. + +``` +authentication=[{enable=true, backend="built-in-database", mechanism="password-based"}] +## below value will replace the whole array, but not to override just one field. +authentication=[{enable=true}] +``` diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 153800414..7ebe7645b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -18,8 +18,9 @@ %% API -export([start_link/0, mnesia/1]). --export([multicall/3, multicall/5, query/1, reset/0, status/0, skip_failed_commit/1]). --export([get_node_tnx_id/1]). +-export([multicall/3, multicall/5, query/1, reset/0, status/0, + skip_failed_commit/1, fast_forward_to_commit/2]). +-export([get_node_tnx_id/1, latest_tnx_id/0]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, handle_continue/2, code_change/3]). @@ -60,21 +61,28 @@ start_link() -> start_link(Node, Name, RetryMs) -> gen_server:start_link({local, Name}, ?MODULE, [Node, RetryMs], []). --spec multicall(Module, Function, Args) -> {ok, TnxId, term()} | {error, Reason} when +%% @doc return {ok, TnxId, MFARes} the first MFA result when all MFA run ok. +%% return {error, MFARes} when the first MFA result is no ok or {ok, term()}. +%% return {retry, TnxId, MFARes, Nodes} when some Nodes failed and some Node ok. +-spec multicall(Module, Function, Args) -> + {ok, TnxId, term()} | {error, Reason} | {retry, TnxId, MFARes, node()} when Module :: module(), Function :: atom(), Args :: [term()], + MFARes :: term(), TnxId :: pos_integer(), Reason :: string(). multicall(M, F, A) -> multicall(M, F, A, all, timer:minutes(2)). --spec multicall(Module, Function, Args, SucceedNum, Timeout) -> {ok, TnxId, term()} |{error, Reason} when +-spec multicall(Module, Function, Args, SucceedNum, Timeout) -> + {ok, TnxId, MFARes} | {error, Reason} | {retry, TnxId, MFARes, node()} when Module :: module(), Function :: atom(), Args :: [term()], SucceedNum :: pos_integer() | all, TnxId :: pos_integer(), + MFARes :: term(), Timeout :: timeout(), Reason :: string(). multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNum >= 1 -> @@ -108,7 +116,10 @@ multicall(M, F, A, RequireNum, Timeout) when RequireNum =:= all orelse RequireNu end, case OkOrFailed of ok -> InitRes; - _ -> OkOrFailed + {error, Error0} -> {error, Error0}; + {retry, Node0} -> + {ok, TnxId0, MFARes} = InitRes, + {retry, TnxId0, MFARes, Node0} end. -spec query(pos_integer()) -> {'atomic', map()} | {'aborted', Reason :: term()}. @@ -122,6 +133,11 @@ reset() -> gen_server:call(?MODULE, reset). status() -> transaction(fun trans_status/0, []). +-spec latest_tnx_id() -> pos_integer(). +latest_tnx_id() -> + {atomic, TnxId} = transaction(fun get_latest_id/0, []), + TnxId. + -spec get_node_tnx_id(node()) -> integer(). get_node_tnx_id(Node) -> case mnesia:wread({?CLUSTER_COMMIT, Node}) of @@ -136,6 +152,13 @@ get_node_tnx_id(Node) -> skip_failed_commit(Node) -> gen_server:call({?MODULE, Node}, skip_failed_commit). +%% Regardless of what MFA is returned, consider it a success), +%% then skip the specified TnxId. +%% If CurrTnxId >= TnxId, nothing happened. +%% If CurrTnxId < TnxId, the CurrTnxId will skip to TnxId. +-spec fast_forward_to_commit(node(), pos_integer()) -> pos_integer(). +fast_forward_to_commit(Node, ToTnxId) -> + gen_server:call({?MODULE, Node}, {fast_forward_to_commit, ToTnxId}). %%%=================================================================== %%% gen_server callbacks %%%=================================================================== @@ -165,8 +188,13 @@ handle_call({initiate, MFA}, _From, State = #{node := Node}) -> {aborted, Reason} -> {reply, {error, Reason}, State, {continue, ?CATCH_UP}} end; -handle_call(skip_failed_commit, _From, State) -> - {reply, ok, State, catch_up(State, true)}; +handle_call(skip_failed_commit, _From, State = #{node := Node}) -> + Timeout = catch_up(State, true), + {atomic, LatestId} = transaction(fun get_node_tnx_id/1, [Node]), + {reply, LatestId, State, Timeout}; +handle_call({fast_forward_to_commit, ToTnxId}, _From, State) -> + NodeId = do_fast_forward_to_commit(ToTnxId, State), + {reply, NodeId, State, catch_up(State)}; handle_call(_, _From, State) -> {reply, ok, State, catch_up(State)}. @@ -245,7 +273,8 @@ do_catch_up(ToTnxId, Node) -> {false, Error} -> mnesia:abort(Error) end; [#cluster_rpc_commit{tnx_id = LastAppliedId}] -> - Reason = lists:flatten(io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", + Reason = lists:flatten( + io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)", [Node, LastAppliedId, ToTnxId])), ?SLOG(error, #{ msg => "catch up failed!", @@ -258,6 +287,20 @@ do_catch_up(ToTnxId, Node) -> commit(Node, TnxId) -> ok = mnesia:write(?CLUSTER_COMMIT, #cluster_rpc_commit{node = Node, tnx_id = TnxId}, write). +do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) -> + {atomic, NodeId} = transaction(fun get_node_tnx_id/1, [Node]), + case NodeId >= ToTnxId of + true -> NodeId; + false -> + {atomic, LatestId} = transaction(fun get_latest_id/0, []), + case LatestId =< NodeId of + true -> NodeId; + false -> + catch_up(State, true), + do_fast_forward_to_commit(ToTnxId, State) + end + end. + get_latest_id() -> case mnesia:last(?CLUSTER_MFA) of '$end_of_table' -> 0; @@ -269,7 +312,8 @@ init_mfa(Node, MFA) -> LatestId = get_latest_id(), ok = do_catch_up_in_one_trans(LatestId, Node), TnxId = LatestId + 1, - MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA, initiator = Node, created_at = erlang:localtime()}, + MFARec = #cluster_rpc_mfa{tnx_id = TnxId, mfa = MFA, + initiator = Node, created_at = erlang:localtime()}, ok = mnesia:write(?CLUSTER_MFA, MFARec, write), ok = commit(Node, TnxId), case apply_mfa(TnxId, MFA) of @@ -344,7 +388,7 @@ wait_for_all_nodes_commit(TnxId, Delay, Remain) -> ok = timer:sleep(Delay), wait_for_all_nodes_commit(TnxId, Delay, Remain - Delay); [] -> ok; - Nodes -> {error, Nodes} + Nodes -> {retry, Nodes} end. wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> @@ -356,7 +400,7 @@ wait_for_nodes_commit(RequiredNum, TnxId, Delay, Remain) -> false -> case lagging_node(TnxId) of [] -> ok; %% All commit but The succeedNum > length(nodes()). - Nodes -> {error, Nodes} + Nodes -> {retry, Nodes} end end. diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index c3dfa8c49..dec07f35c 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -16,6 +16,7 @@ -module(emqx_conf). -compile({no_auto_import, [get/1, get/2]}). +-include_lib("emqx/include/logger.hrl"). -export([add_handler/2, remove_handler/1]). -export([get/1, get/2, get_raw/2, get_all/1]). @@ -23,6 +24,7 @@ -export([update/3, update/4]). -export([remove/2, remove/3]). -export([reset/2, reset/3]). +-export([gen_doc/1]). %% for rpc -export([get_node_and_config/1]). @@ -122,14 +124,29 @@ reset(Node, KeyPath, Opts) when Node =:= node() -> reset(Node, KeyPath, Opts) -> rpc:call(Node, ?MODULE, reset, [KeyPath, Opts]). +-spec gen_doc(file:name_all()) -> ok. +gen_doc(File) -> + Version = emqx_release:version(), + Title = "# EMQ X " ++ Version ++ " Configuration", + BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]), + {ok, Body} = file:read_file(BodyFile), + Doc = hocon_schema_doc:gen(emqx_conf_schema, #{title => Title, + body => Body}), + file:write_file(File, Doc). + %%-------------------------------------------------------------------- -%% Internal funcs +%% Internal functions %%-------------------------------------------------------------------- multicall(M, F, Args) -> case emqx_cluster_rpc:multicall(M, F, Args) of - {ok, _TnxId, Res} -> + {ok, _TnxId, Res} -> Res; + {retry, TnxId, Res, Nodes} -> + %% The init MFA return ok, but other nodes failed. + %% We return ok and alert an alarm. + ?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes, + tnx_id => TnxId, mfa => {M, F, Args}}), Res; - {error, Reason} -> - {error, Reason} + {error, Error} -> %% all MFA return not ok or {ok, term()}. + Error end. diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl new file mode 100644 index 000000000..7fb421e75 --- /dev/null +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_conf_cli). +-export([ load/0 + , admins/1 + , unload/0 + ]). + +-define(CMD, cluster_call). + +load() -> + emqx_ctl:register_command(?CMD, {?MODULE, admins}, []). + +unload() -> + emqx_ctl:unregister_command(?CMD). + +admins(["status"]) -> status(); + +admins(["skip"]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + lists:foreach(fun emqx_cluster_rpc:skip_failed_commit/1, Nodes), + status(); + +admins(["skip", Node0]) -> + status(), + Node = list_to_existing_atom(Node0), + emqx_cluster_rpc:skip_failed_commit(Node), + status(); + +admins(["tnxid", TnxId0]) -> + TnxId = list_to_integer(TnxId0), + emqx_ctl:print("~p~n", [emqx_cluster_rpc:query(TnxId)]); + +admins(["fast_forward"]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + TnxId = emqx_cluster_rpc:latest_tnx_id(), + lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes), + status(); + +admins(["fast_forward", ToTnxId]) -> + status(), + Nodes = mria_mnesia:running_nodes(), + TnxId = list_to_integer(ToTnxId), + lists:foreach(fun(N) -> emqx_cluster_rpc:fast_forward_to_commit(N, TnxId) end, Nodes), + status(); + +admins(["fast_forward", Node0, ToTnxId]) -> + status(), + TnxId = list_to_integer(ToTnxId), + Node = list_to_existing_atom(Node0), + emqx_cluster_rpc:fast_forward_to_commit(Node, TnxId), + status(); + +admins(_) -> + emqx_ctl:usage( + [ + {"cluster_call status", "status"}, + {"cluster_call skip [node]", "increase one commit on specific node"}, + {"cluster_call tnxid ", "get detailed about TnxId"}, + {"cluster_call fast_forward [node] [tnx_id]", "fast forwards to tnx_id" } + ]). + +status() -> + emqx_ctl:print("-----------------------------------------------\n"), + {atomic, Status} = emqx_cluster_rpc:status(), + lists:foreach(fun(S) -> + #{ + node := Node, + tnx_id := TnxId, + mfa := {M, F, A}, + created_at := CreatedAt + } = S, + emqx_ctl:print("~p:[~w] CreatedAt:~p ~p:~p/~w\n", + [Node, TnxId, CreatedAt, M, F, length(A)]) + end, Status), + emqx_ctl:print("-----------------------------------------------\n"). diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d8bb2423b..8103796b7 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -24,6 +24,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -62,8 +63,12 @@ namespace() -> undefined. roots() -> - %% authorization configs are merged in THIS schema's "authorization" fields - lists:keydelete("authorization", 1, emqx_schema:roots(high)) ++ + PtKey = ?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, + case persistent_term:get(PtKey, undefined) of + undefined -> persistent_term:put(PtKey, emqx_authn_schema); + _ -> ok + end, + emqx_schema_high_prio_roots() ++ [ {"node", sc(hoconsc:ref("node"), #{ desc => "Node name, cookie, config & data directories " @@ -87,20 +92,6 @@ roots() -> "should work, but in case you need to do performance " "fine-turning or experiment a bit, this is where to look." })} - , {"authorization", - sc(hoconsc:ref("authorization"), - #{ desc => """ -Authorization a.k.a ACL.
-In EMQ X, MQTT client access control is extremly flexible.
-An out of the box set of authorization data sources are supported. -For example,
-'file' source is to support concise and yet generic ACL rules in a file;
-'built-in-database' source can be used to store per-client customisable rule sets, -natively in the EMQ X node;
-'http' source to make EMQ X call an external HTTP API to make the decision;
-'postgresql' etc. to look up clients or rules from external databases;
-""" - })} , {"db", sc(ref("db"), #{ desc => "Settings of the embedded database." @@ -251,14 +242,12 @@ fields("node") -> [ {"name", sc(string(), #{ default => "emqx@127.0.0.1" - , override_env => "EMQX_NODE_NAME" })} , {"cookie", sc(string(), #{ mapping => "vm_args.-setcookie", default => "emqxsecretcookie", - sensitive => true, - override_env => "EMQX_NODE_COOKIE" + sensitive => true })} , {"data_dir", sc(string(), @@ -275,9 +264,25 @@ fields("node") -> #{ mapping => "emqx_machine.global_gc_interval" , default => "15m" })} - , {"crash_dump_dir", + , {"crash_dump_file", sc(file(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP" + , desc => "Location of the crash dump file" + })} + , {"crash_dump_seconds", + sc(emqx_schema:duration_s(), + #{ mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS" + , default => "30s" + , desc => """ +The number of seconds that the broker is allowed to spend writing +a crash dump +""" + })} + , {"crash_dump_bytes", + sc(emqx_schema:bytesize(), + #{ mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES" + , default => "100MB" + , desc => "The maximum size of a crash dump file in bytes." })} , {"dist_net_ticktime", sc(emqx_schema:duration(), @@ -347,6 +352,23 @@ to rlog. List of core nodes that the replicant will connect to.
Note: this parameter only takes effect when the backend is set to rlog and the role is set to replicant. +""" + })} + , {"rpc_module", + sc(hoconsc:enum([gen_rpc, rpc]), + #{ mapping => "mria.rlog_rpc_module" + , default => gen_rpc + , desc => """ +Protocol used for pushing transaction logs to the replicant nodes. +""" + })} + , {"tlog_push_mode", + sc(hoconsc:enum([sync, async]), + #{ mapping => "mria.tlog_push_mode" + , default => async + , desc => """ +In sync mode the core node waits for an ack from the replicant nodes before sending the next +transaction log entry. """ })} ]; @@ -812,3 +834,22 @@ ensure_list(V) -> roots(Module) -> lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)). + +%% Like authentication schema, authorization schema is incomplete in emqx_schema +%% module, this function replaces the root filed "authorization" with a new schema +emqx_schema_high_prio_roots() -> + Roots = emqx_schema:roots(high), + Authz = {"authorization", + sc(hoconsc:ref("authorization"), + #{ desc => """ +Authorization a.k.a ACL.
+In EMQ X, MQTT client access control is extremly flexible.
+An out of the box set of authorization data sources are supported. +For example,
+'file' source is to support concise and yet generic ACL rules in a file;
+'built-in-database' source can be used to store per-client customisable rule sets, +natively in the EMQ X node;
+'http' source to make EMQ X call an external HTTP API to make the decision;
+'postgresql' etc. to look up clients or rules from external databases;
+""" })}, + lists:keyreplace("authorization", 1, Roots, Authz). diff --git a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl index 993ab3dc5..ad74faf99 100644 --- a/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl +++ b/apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl @@ -33,7 +33,8 @@ all() -> [ t_commit_ok_but_apply_fail_on_other_node, t_commit_ok_apply_fail_on_other_node_then_recover, t_del_stale_mfa, - t_skip_failed_commit + t_skip_failed_commit, + t_fast_forward_commit ]. suite() -> [{timetrap, {minutes, 3}}]. groups() -> []. @@ -183,13 +184,37 @@ t_skip_failed_commit(_Config) -> ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], tnx_ids(List1)), {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, - {ok, _, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), - ok = gen_server:call(?NODE2, skip_failed_commit, 5000), + {ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + 2 = gen_server:call(?NODE2, skip_failed_commit, 5000), {atomic, List2} = emqx_cluster_rpc:status(), ?assertEqual([{Node, 2}, {{Node, ?NODE2}, 2}, {{Node, ?NODE3}, 1}], tnx_ids(List2)), ok. +t_fast_forward_commit(_Config) -> + emqx_cluster_rpc:reset(), + {atomic, []} = emqx_cluster_rpc:status(), + {ok, 1, ok} = emqx_cluster_rpc:multicall(io, format, ["test~n"], all, 1000), + ct:sleep(180), + {atomic, List1} = emqx_cluster_rpc:status(), + Node = node(), + ?assertEqual([{Node, 1}, {{Node, ?NODE2}, 1}, {{Node, ?NODE3}, 1}], + tnx_ids(List1)), + {M, F, A} = {?MODULE, failed_on_node, [erlang:whereis(?NODE1)]}, + {ok, 2, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 3, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 4, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {ok, 5, ok} = emqx_cluster_rpc:multicall(M, F, A, 1, 1000), + {retry, 6, ok, _} = emqx_cluster_rpc:multicall(M, F, A, 2, 1000), + 3 = gen_server:call(?NODE2, {fast_forward_to_commit, 3}, 5000), + 4 = gen_server:call(?NODE2, {fast_forward_to_commit, 4}, 5000), + 6 = gen_server:call(?NODE2, {fast_forward_to_commit, 7}, 5000), + 2 = gen_server:call(?NODE3, {fast_forward_to_commit, 2}, 5000), + {atomic, List2} = emqx_cluster_rpc:status(), + ?assertEqual([{Node, 6}, {{Node, ?NODE2}, 6}, {{Node, ?NODE3}, 2}], + tnx_ids(List2)), + ok. + tnx_ids(Status) -> lists:sort(lists:map(fun(#{tnx_id := TnxId, node := Node}) -> {Node, TnxId} end, Status)). diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf index 06395ac94..8929598be 100644 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -1,4 +1,5 @@ #connectors.mqtt.my_mqtt_connector { +# mode = cluster_shareload # server = "127.0.0.1:1883" # proto_ver = "v4" # username = "username1" @@ -8,7 +9,6 @@ # retry_interval = "30s" # max_inflight = 32 # reconnect_interval = "30s" -# bridge_mode = true # replayq { # dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" # seg_bytes = "100MB" diff --git a/apps/emqx_connector/src/emqx_connector_api.erl b/apps/emqx_connector/src/emqx_connector_api.erl index 6eb397519..95bc33a83 100644 --- a/apps/emqx_connector/src/emqx_connector_api.erl +++ b/apps/emqx_connector/src/emqx_connector_api.erl @@ -30,6 +30,8 @@ %% API callbacks -export(['/connectors_test'/2, '/connectors'/2, '/connectors/:id'/2]). +-define(CONN_TYPES, [mqtt]). + -define(TRY_PARSE_ID(ID, EXPR), try emqx_connector:parse_connector_id(Id) of {ConnType, ConnName} -> @@ -38,7 +40,7 @@ catch error:{invalid_bridge_id, Id0} -> {400, #{code => 'INVALID_ID', message => <<"invalid_bridge_id: ", Id0/binary, - ". Bridge Ids must be of format :">>}} + ". Bridge Ids must be of format {type}:{name}">>}} end). namespace() -> "connector". @@ -53,17 +55,71 @@ error_schema(Code, Message) -> , {message, mk(string(), #{example => Message})} ]. -connector_info() -> - hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector_info") - ]). +put_request_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_connector_schema:put_request(), connector_info_examples(put)). -connector_test_info() -> - hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector_test_info") - ]). +post_request_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_connector_schema:post_request(), connector_info_examples(post)). -connector_req() -> - hoconsc:union([ ref(emqx_connector_schema, "mqtt_connector") - ]). +get_response_body_schema() -> + emqx_dashboard_swagger:schema_with_examples( + emqx_connector_schema:get_response(), connector_info_examples(get)). + +connector_info_array_example(Method) -> + [Config || #{value := Config} <- maps:values(connector_info_examples(Method))]. + +connector_info_examples(Method) -> + lists:foldl(fun(Type, Acc) -> + SType = atom_to_list(Type), + maps:merge(Acc, #{ + Type => #{ + summary => bin(string:uppercase(SType) ++ " Connector"), + value => info_example(Type, Method) + } + }) + end, #{}, ?CONN_TYPES). + +info_example(Type, Method) -> + maps:merge(info_example_basic(Type), + method_example(Type, Method)). + +method_example(Type, get) -> + SType = atom_to_list(Type), + SName = "my_" ++ SType ++ "_connector", + #{ + id => bin(SType ++ ":" ++ SName), + type => bin(SType), + name => bin(SName) + }; +method_example(Type, post) -> + SType = atom_to_list(Type), + SName = "my_" ++ SType ++ "_connector", + #{ + type => bin(SType), + name => bin(SName) + }; +method_example(_Type, put) -> + #{}. + +info_example_basic(mqtt) -> + #{ + mode => cluster_shareload, + server => <<"127.0.0.1:1883">>, + reconnect_interval => <<"30s">>, + proto_ver => <<"v4">>, + username => <<"foo">>, + password => <<"bar">>, + clientid => <<"foo">>, + clean_start => true, + keepalive => <<"300s">>, + retry_interval => <<"30s">>, + max_inflight => 100, + ssl => #{ + enable => false + } + }. param_path_id() -> [{id, mk(binary(), #{in => path, example => <<"mqtt:my_mqtt_connector">>})}]. @@ -74,9 +130,9 @@ schema("/connectors_test") -> post => #{ tags => [<<"connectors">>], description => <<"Test creating a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format '{type}:{name}'">>, summary => <<"Test creating connector">>, - requestBody => connector_test_info(), + requestBody => post_request_body_schema(), responses => #{ 200 => <<"Test connector OK">>, 400 => error_schema('TEST_FAILED', "connector test failed") @@ -92,17 +148,19 @@ schema("/connectors") -> description => <<"List all connectors">>, summary => <<"List connectors">>, responses => #{ - 200 => mk(array(connector_info()), #{desc => "List of connectors"}) + 200 => emqx_dashboard_swagger:schema_with_example( + array(emqx_connector_schema:get_response()), + connector_info_array_example(get)) } }, post => #{ tags => [<<"connectors">>], description => <<"Create a new connector by given Id
" - "The Id must be of format :">>, + "The ID must be of format '{type}:{name}'">>, summary => <<"Create connector">>, - requestBody => connector_info(), + requestBody => post_request_body_schema(), responses => #{ - 201 => connector_info(), + 201 => get_response_body_schema(), 400 => error_schema('ALREADY_EXISTS', "connector already exists") } } @@ -117,7 +175,7 @@ schema("/connectors/:id") -> summary => <<"Get connector">>, parameters => param_path_id(), responses => #{ - 200 => connector_info(), + 200 => get_response_body_schema(), 404 => error_schema('NOT_FOUND', "Connector not found") } }, @@ -126,9 +184,9 @@ schema("/connectors/:id") -> description => <<"Update an existing connector by Id">>, summary => <<"Update connector">>, parameters => param_path_id(), - requestBody => connector_req(), + requestBody => put_request_body_schema(), responses => #{ - 200 => <<"Update connector successfully">>, + 200 => get_response_body_schema(), 400 => error_schema('UPDATE_FAIL', "Update failed"), 404 => error_schema('NOT_FOUND', "Connector not found") }}, @@ -143,8 +201,8 @@ schema("/connectors/:id") -> }} }. -'/connectors_test'(post, #{body := #{<<"bridge_type">> := ConnType} = Params}) -> - case emqx_connector:create_dry_run(ConnType, maps:remove(<<"bridge_type">>, Params)) of +'/connectors_test'(post, #{body := #{<<"type">> := ConnType} = Params}) -> + case emqx_connector:create_dry_run(ConnType, maps:remove(<<"type">>, Params)) of ok -> {200}; {error, Error} -> {400, error_msg('BAD_ARG', Error)} @@ -153,17 +211,20 @@ schema("/connectors/:id") -> '/connectors'(get, _Request) -> {200, emqx_connector:list()}; -'/connectors'(post, #{body := #{<<"id">> := Id} = Params}) -> - ?TRY_PARSE_ID(Id, - case emqx_connector:lookup(ConnType, ConnName) of - {ok, _} -> - {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; - {error, not_found} -> - case emqx_connector:update(ConnType, ConnName, maps:remove(<<"id">>, Params)) of - {ok, #{raw_config := RawConf}} -> {201, RawConf#{<<"id">> => Id}}; - {error, Error} -> {400, error_msg('BAD_ARG', Error)} - end - end). +'/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) -> + ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()), + case emqx_connector:lookup(ConnType, ConnName) of + {ok, _} -> + {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)}; + {error, not_found} -> + case emqx_connector:update(ConnType, ConnName, + maps:without([<<"type">>, <<"name">>], Params)) of + {ok, #{raw_config := RawConf}} -> + {201, RawConf#{<<"id">> => + emqx_connector:connector_id(ConnType, ConnName)}}; + {error, Error} -> {400, error_msg('BAD_ARG', Error)} + end + end. '/connectors/:id'(get, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID(Id, @@ -200,4 +261,7 @@ schema("/connectors/:id") -> error_msg(Code, Msg) when is_binary(Msg) -> #{code => Code, message => Msg}; error_msg(Code, Msg) -> - #{code => Code, message => list_to_binary(io_lib:format("~p", [Msg]))}. + #{code => Code, message => bin(io_lib:format("~p", [Msg]))}. + +bin(S) when is_list(S) -> + list_to_binary(S). diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 11eac9c91..6a1b15e57 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -118,7 +118,7 @@ on_start(InstId, Config = #{mongo_type := Type, false -> [{ssl, false}] end, Topology = maps:get(topology, NConfig, #{}), - Opts = [{type, init_type(NConfig)}, + Opts = [{mongo_type, init_type(NConfig)}, {hosts, Hosts}, {pool_size, PoolSize}, {options, init_topology_options(maps:to_list(Topology), [])}, @@ -187,6 +187,7 @@ connect(Opts) -> WorkerOptions = proplists:get_value(worker_options, Opts, []), mongo_api:connect(Type, Hosts, Options, WorkerOptions). + mongo_query(Conn, find, Collection, Selector, Projector) -> mongo_api:find(Conn, Collection, Selector, Projector); @@ -268,7 +269,7 @@ srv_record(_) -> undefined. parse_servers(Type, Servers) when is_binary(Servers) -> parse_servers(Type, binary_to_list(Servers)); parse_servers(Type, Servers) when is_list(Servers) -> - case string:split(Servers, ",", trailing) of + case string:split(Servers, ",", all) of [Host | _] when Type =:= single -> [Host]; Hosts -> diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 1acd8b298..6bc609fa8 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -40,6 +40,8 @@ -behaviour(hocon_schema). +-import(hoconsc, [mk/2]). + -export([ roots/0 , fields/1]). @@ -49,7 +51,25 @@ roots() -> fields("config"). fields("config") -> - emqx_connector_mqtt_schema:fields("config"). + emqx_connector_mqtt_schema:fields("config"); + +fields("get") -> + [{id, mk(binary(), + #{ desc => "The connector Id" + , example => <<"mqtt:my_mqtt_connector">> + })}] + ++ fields("post"); + +fields("put") -> + emqx_connector_mqtt_schema:fields("connector"); + +fields("post") -> + [ {type, mk(mqtt, #{desc => "The Connector Type"})} + , {name, mk(binary(), + #{ desc => "The Connector Name" + , example => <<"my_mqtt_connector">> + })} + ] ++ fields("put"). %% =================================================================== %% supervisor APIs @@ -100,7 +120,7 @@ on_start(InstId, Conf) -> BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ name => InstanceId, - clientid => clientid(InstanceId), + clientid => clientid(maps:get(clientid, Conf, InstId)), subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)), forwards => make_forward_confs(maps:get(egress, Conf, undefined)) }, @@ -162,7 +182,6 @@ basic_config(#{ server := Server, reconnect_interval := ReconnIntv, proto_ver := ProtoVer, - bridge_mode := BridgeMod, username := User, password := Password, clean_start := CleanStart, @@ -177,7 +196,7 @@ basic_config(#{ server => Server, reconnect_interval => ReconnIntv, proto_ver => ProtoVer, - bridge_mode => BridgeMod, + bridge_mode => true, username => User, password => Password, clean_start => CleanStart, @@ -190,4 +209,4 @@ basic_config(#{ }. clientid(Id) -> - list_to_binary(lists:concat([Id, ":", node()])). + iolist_to_binary([Id, ":", atom_to_list(node())]). diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index 264a6dbd6..c386a829f 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -4,33 +4,47 @@ -include_lib("typerefl/include/types.hrl"). +-import(hoconsc, [mk/2, ref/2]). + -export([roots/0, fields/1]). +-export([ get_response/0 + , put_request/0 + , post_request/0 + ]). + +-define(CONN_TYPES, [mqtt]). + +%%====================================================================================== +%% For HTTP APIs + +get_response() -> + http_schema("get"). + +put_request() -> + http_schema("put"). + +post_request() -> + http_schema("post"). + +http_schema(Method) -> + Schemas = [ref(schema_mod(Type), Method) || Type <- ?CONN_TYPES], + hoconsc:union(Schemas). + %%====================================================================================== %% Hocon Schema Definitions roots() -> ["connectors"]. +fields(connectors) -> fields("connectors"); fields("connectors") -> [ {mqtt, - sc(hoconsc:map(name, - hoconsc:union([ ref("mqtt_connector") + mk(hoconsc:map(name, + hoconsc:union([ ref(emqx_connector_mqtt_schema, "connector") ])), #{ desc => "MQTT bridges" })} - ]; + ]. -fields("mqtt_connector") -> - emqx_connector_mqtt_schema:fields("connector"); - -fields("mqtt_connector_info") -> - [{id, sc(binary(), #{desc => "The connector Id"})}] - ++ fields("mqtt_connector"); - -fields("mqtt_connector_test_info") -> - [{bridge_type, sc(mqtt, #{desc => "The Bridge Type"})}] - ++ fields("mqtt_connector"). - -sc(Type, Meta) -> hoconsc:mk(Type, Meta). - -ref(Field) -> hoconsc:ref(?MODULE, Field). +schema_mod(Type) -> + list_to_atom(lists:concat(["emqx_connector_", Type])). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 6436a4c96..2338129d1 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -8,7 +8,7 @@ %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, +%% cluster_shareload under the License is cluster_shareload on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. @@ -38,7 +38,24 @@ fields("config") -> topic_mappings(); fields("connector") -> - [ {server, + [ {mode, + sc(hoconsc:enum([cluster_singleton, cluster_shareload]), + #{ default => cluster_shareload + , desc => """ +The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'
+ +- cluster_singleton: create an unique MQTT connection within the emqx cluster.
+In 'cluster_singleton' node, all messages toward the remote broker go through the same +MQTT connection.
+- cluster_shareload: create an MQTT connection on each node in the emqx cluster.
+In 'cluster_shareload' mode, the incomming load from the remote broker is shared by +using shared subscription.
+Note that the 'clientid' is suffixed by the node name, this is to avoid +clientid conflicts between different nodes. And we can only use shared subscription +topic filters for 'from_remote_topic'. +""" + })} + , {server, sc(emqx_schema:ip_port(), #{ default => "127.0.0.1:1883" , desc => "The host and port of the remote MQTT broker" @@ -49,11 +66,6 @@ fields("connector") -> #{ default => v4 , desc => "The MQTT protocol version" })} - , {bridge_mode, - sc(boolean(), - #{ default => true - , desc => "The bridge mode of the MQTT protocol" - })} , {username, sc(binary(), #{ default => "emqx" @@ -66,8 +78,7 @@ fields("connector") -> })} , {clientid, sc(binary(), - #{ default => "emqx_${nodename}" - , desc => "The clientid of the MQTT protocol" + #{ desc => "The clientid of the MQTT protocol" })} , {clean_start, sc(boolean(), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 96f530563..bbac76674 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -24,7 +24,11 @@ -define(CONF_DEFAULT, <<"connectors: {}">>). -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). +-define(CONNECTR_TYPE, <<"mqtt">>). +-define(CONNECTR_NAME, <<"test_connector">>). -define(CONNECTR_ID, <<"mqtt:test_connector">>). +-define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>). +-define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>). -define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>). -define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>). -define(MQTT_CONNECOTR(Username), @@ -63,8 +67,8 @@ -define(metrics(MATCH, SUCC, FAILED, SPEED, SPEED5M, SPEEDMAX), #{<<"matched">> := MATCH, <<"success">> := SUCC, - <<"failed">> := FAILED, <<"speed">> := SPEED, - <<"speed_last5m">> := SPEED5M, <<"speed_max">> := SPEEDMAX}). + <<"failed">> := FAILED, <<"rate">> := SPEED, + <<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -115,7 +119,9 @@ t_mqtt_crud_apis(_) -> %% POST /connectors/ will create a connector User1 = <<"user1">>, {ok, 201, Connector} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}), + ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), %ct:pal("---connector: ~p", [Connector]), ?assertMatch(#{ <<"id">> := ?CONNECTR_ID @@ -128,7 +134,9 @@ t_mqtt_crud_apis(_) -> %% create a again returns an error {ok, 400, RetMsg} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}), + ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), ?assertMatch( #{ <<"code">> := _ , <<"message">> := <<"connector already exists">> @@ -187,7 +195,9 @@ t_mqtt_conn_bridge_ingress(_) -> %% then we add a mqtt connector, using POST User1 = <<"user1">>, {ok, 201, Connector} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}), + ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), %ct:pal("---connector: ~p", [Connector]), ?assertMatch(#{ <<"id">> := ?CONNECTR_ID @@ -201,11 +211,14 @@ t_mqtt_conn_bridge_ingress(_) -> %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_INGRESS}), + ?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_INGRESS + }), %ct:pal("---bridge: ~p", [Bridge]), ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS - , <<"bridge_type">> := <<"mqtt">> + , <<"type">> := <<"mqtt">> , <<"status">> := <<"connected">> , <<"connector">> := ?CONNECTR_ID }, jsx:decode(Bridge)), @@ -250,7 +263,9 @@ t_mqtt_conn_bridge_egress(_) -> %% then we add a mqtt connector, using POST User1 = <<"user1">>, {ok, 201, Connector} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR(User1)#{<<"id">> => ?CONNECTR_ID}), + ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), %ct:pal("---connector: ~p", [Connector]), ?assertMatch(#{ <<"id">> := ?CONNECTR_ID @@ -264,11 +279,15 @@ t_mqtt_conn_bridge_egress(_) -> %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_EGRESS}), + ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }), %ct:pal("---bridge: ~p", [Bridge]), ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS - , <<"bridge_type">> := <<"mqtt">> + , <<"type">> := ?CONNECTR_TYPE + , <<"name">> := ?BRIDGE_NAME_EGRESS , <<"status">> := <<"connected">> , <<"connector">> := ?CONNECTR_ID }, jsx:decode(Bridge)), @@ -322,7 +341,10 @@ t_mqtt_conn_update(_) -> %% then we add a mqtt connector, using POST {ok, 201, Connector} = request(post, uri(["connectors"]), - ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{<<"id">> => ?CONNECTR_ID}), + ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>) + #{ <<"type">> => ?CONNECTR_TYPE + , <<"name">> => ?CONNECTR_NAME + }), %ct:pal("---connector: ~p", [Connector]), ?assertMatch(#{ <<"id">> := ?CONNECTR_ID @@ -332,9 +354,13 @@ t_mqtt_conn_update(_) -> %% ... and a MQTT bridge, using POST %% we bind this bridge to the connector created just now {ok, 201, Bridge} = request(post, uri(["bridges"]), - ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{<<"id">> => ?BRIDGE_ID_EGRESS}), + ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }), ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS - , <<"bridge_type">> := <<"mqtt">> + , <<"type">> := <<"mqtt">> + , <<"name">> := ?BRIDGE_NAME_EGRESS , <<"status">> := <<"connected">> , <<"connector">> := ?CONNECTR_ID }, jsx:decode(Bridge)), @@ -358,9 +384,15 @@ t_mqtt_conn_testing(_) -> %% APIs for testing the connectivity %% then we add a mqtt connector, using POST {ok, 200, <<>>} = request(post, uri(["connectors_test"]), - ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{<<"bridge_type">> => <<"mqtt">>}), + ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }), {ok, 400, _} = request(post, uri(["connectors_test"]), - ?MQTT_CONNECOTR2(<<"127.0.0.1:2883">>)#{<<"bridge_type">> => <<"mqtt">>}). + ?MQTT_CONNECOTR2(<<"127.0.0.1:2883">>)#{ + <<"type">> => ?CONNECTR_TYPE, + <<"name">> => ?BRIDGE_NAME_EGRESS + }). %%-------------------------------------------------------------------- %% HTTP Request diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 59c0f560a..a722872a3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -182,12 +182,12 @@ check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, case hocon_schema:field_schema(Type, in) of path -> - Option = #{atom_key => true, override_env => false}, + Option = #{atom_key => true}, NewBindings = hocon_schema:check_plain(Schema, Bindings, Option), NewBindingsAcc = maps:merge(BindingsAcc, NewBindings), check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc); query -> - Option = #{override_env => false}, + Option = #{}, NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option), NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr), check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc) @@ -201,7 +201,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) -> _ -> Type0 end, NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]}, - Option = #{override_env => false, nullable => true}, + Option = #{nullable => true}, #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option), NewBody; %% TODO not support nest object check yet, please use ref! @@ -214,7 +214,7 @@ check_request_body(#{body := Body}, Schema, Module, CheckFun, true) -> check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) -> lists:foldl(fun({Name, Type}, Acc) -> Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]}, - maps:merge(Acc, CheckFun(Schema, Body, #{override_env => false})) + maps:merge(Acc, CheckFun(Schema, Body, #{})) end, #{}, Spec). %% tags, description, summary, security, deprecated @@ -337,19 +337,28 @@ components(Refs) -> components([], SpecAcc, []) -> SpecAcc; components([], SpecAcc, SubRefAcc) -> components(SubRefAcc, SpecAcc, []); components([{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> - Props = apply(Module, fields, [Field]), + Props = hocon_schema_fields(Module, Field), Namespace = namespace(Module), {Object, SubRefs} = parse_object(Props, Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Object}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc); %% parameters in ref only have one value, not array components([{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> - Props = apply(Module, fields, [Field]), + Props = hocon_schema_fields(Module, Field), {[Param], SubRefs} = parameters(Props, Module), Namespace = namespace(Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). +hocon_schema_fields(Module, StructName) -> + case apply(Module, fields, [StructName]) of + #{fields := Fields, desc := _} -> + %% evil here, as it's match hocon_schema's internal representation + Fields; %% TODO: make use of desc ? + Other -> + Other + end. + %% Semantic error at components.schemas.xxx:xx:xx %% Component names can only contain the characters A-Z a-z 0-9 - . _ %% So replace ':' by '-'. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index ae74fc08e..84cac6229 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -7,7 +7,7 @@ -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_object/1, t_nest_object/1, t_api_spec/1, t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, - t_ref_array_with_key/1, t_ref_array_without_key/1 + t_ref_array_with_key/1, t_ref_array_without_key/1, t_sub_fields/1 ]). -export([ t_object_trans/1, t_object_notrans/1, t_nest_object_trans/1, t_local_ref_trans/1, @@ -154,6 +154,17 @@ t_none_ref(_Config) -> emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path)), ok. +t_sub_fields(_Config) -> + Spec = #{ + post => #{parameters => [], + requestBody => #{<<"content">> => #{<<"application/json">> => + #{<<"schema">> => #{<<"$ref">> => + <<"#/components/schemas/emqx_swagger_requestBody_SUITE.sub_fields">>}}}}, + responses => #{<<"200">> => #{description => <<"ok">>}}}}, + Refs = [{?MODULE, sub_fields}], + validate("/fields/sub", Spec, Refs), + ok. + t_bad_ref(_Config) -> Path = "/ref/bad", Spec = #{ @@ -483,7 +494,7 @@ trans_requestBody(Path, Body, Filter) -> api_spec() -> emqx_dashboard_swagger:spec(?MODULE). paths() -> - ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", + ["/object", "/nest/object", "/ref/local", "/ref/nest/ref", "/fields/sub", "/ref/array/with/key", "/ref/array/without/key"]. schema("/object") -> @@ -506,6 +517,8 @@ schema("/nest/object") -> ]); schema("/ref/local") -> to_schema(mk(hoconsc:ref(good_ref), #{})); +schema("/fields/sub") -> + to_schema(mk(hoconsc:ref(sub_fields), #{})); schema("/ref/remote") -> to_schema(mk(hoconsc:ref(emqx_swagger_remote_schema, "ref2"), #{})); schema("/ref/bad") -> @@ -544,4 +557,20 @@ fields(bad_ref) -> %% don't support maps #{ username => mk(string(), #{}), is_admin => mk(boolean(), #{}) - }. + }; +fields(sub_fields) -> + #{fields => [ + {enable, fun enable/1}, + {init_file, fun init_file/1} + ], + desc => <<"test sub fields">>}. + +enable(type) -> boolean(); +enable(desc) -> <<"Whether to enable tls psk support">>; +enable(default) -> false; +enable(_) -> undefined. + +init_file(type) -> binary(); +init_file(desc) -> <<"test test desc">>; +init_file(nullable) -> true; +init_file(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 4d4a9413e..b4785cf1a 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -14,7 +14,7 @@ -export([paths/0, api_spec/0, schema/1, fields/1]). -export([t_simple_binary/1, t_object/1, t_nest_object/1, t_empty/1, t_error/1, t_raw_local_ref/1, t_raw_remote_ref/1, t_hocon_schema_function/1, t_complicated_type/1, - t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, + t_local_ref/1, t_remote_ref/1, t_bad_ref/1, t_none_ref/1, t_nest_ref/1, t_sub_fields/1, t_ref_array_with_key/1, t_ref_array_without_key/1, t_api_spec/1]). all() -> [{group, spec}]. @@ -23,7 +23,7 @@ groups() -> [ {spec, [parallel], [ t_api_spec, t_simple_binary, t_object, t_nest_object, t_error, t_complicated_type, t_raw_local_ref, t_raw_remote_ref, t_empty, t_hocon_schema_function, - t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, + t_local_ref, t_remote_ref, t_bad_ref, t_none_ref, t_sub_fields, t_ref_array_with_key, t_ref_array_without_key, t_nest_ref]} ]. @@ -163,6 +163,14 @@ t_nest_ref(_Config) -> validate(Path, Object, ExpectRefs), ok. +t_sub_fields(_Config) -> + Path = "/fields/sub", + Object = #{<<"content">> => #{<<"application/json">> => #{<<"schema">> => #{ + <<"$ref">> => <<"#/components/schemas/emqx_swagger_response_SUITE.sub_fields">>}}}}, + ExpectRefs = [{?MODULE, sub_fields}], + validate(Path, Object, ExpectRefs), + ok. + t_complicated_type(_Config) -> Path = "/ref/complicated_type", Object = #{<<"content">> => #{<<"application/json">> => @@ -366,7 +374,9 @@ schema("/ref/complicated_type") -> {fix_integer, hoconsc:mk(typerefl:integer(100), #{})} ] }} - }. + }; +schema("/fields/sub") -> + to_schema(hoconsc:ref(sub_fields)). validate(Path, ExpectObject, ExpectRefs) -> {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path), @@ -400,4 +410,20 @@ fields(bad_ref) -> %% don't support maps #{ username => mk(string(), #{}), is_admin => mk(boolean(), #{}) - }. + }; +fields(sub_fields) -> + #{fields => [ + {enable, fun enable/1}, + {init_file, fun init_file/1} + ], + desc => <<"test sub fields">>}. + +enable(type) -> boolean(); +enable(desc) -> <<"Whether to enable tls psk support">>; +enable(default) -> false; +enable(_) -> undefined. + +init_file(type) -> binary(); +init_file(desc) -> <<"test test desc">>; +init_file(nullable) -> true; +init_file(_) -> undefined. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 43f16f149..e1dad1dee 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -18,9 +18,6 @@ -behaviour(emqx_gateway_channel). --include_lib("emqx/include/logger.hrl"). --include("emqx_coap.hrl"). - %% API -export([ info/1 , info/2 @@ -44,6 +41,12 @@ -export_type([channel/0]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). + -record(channel, { %% Context ctx :: emqx_gateway_ctx:context(), @@ -98,10 +101,10 @@ info(ctx, #channel{ctx = Ctx}) -> stats(_) -> []. -init(ConnInfo = #{peername := {PeerHost, _}, - sockname := {_, SockPort}}, +init(ConnInfoT = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> - Peercert = maps:get(peercert, ConnInfo, undefined), + Peercert = maps:get(peercert, ConnInfoT, undefined), Mountpoint = maps:get(mountpoint, Config, <<>>), ListenerId = case maps:get(listener, Config, undefined) of undefined -> undefined; @@ -123,6 +126,10 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + %% because it is possible to disconnect after init, and then trigger the $event.disconnected hook + %% and these two fields are required in the hook + ConnInfo = ConnInfoT#{proto_name => <<"CoAP">>, proto_ver => <<"1">>}, + Heartbeat = ?GET_IDLE_TIME(Config), #channel{ ctx = Ctx , conninfo = ConnInfo @@ -279,7 +286,7 @@ try_takeover(idle, DesireId, Msg, Channel) -> %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - case emqx_conf:get([gateway, coap, authentication], undefined) of + case emqx_conf:get([gateway, coap, ?AUTHN], undefined) of undefined -> call_session(handle_request, Msg, Channel); _ -> @@ -349,8 +356,6 @@ ensure_connected(Channel = #channel{ctx = Ctx, conninfo = ConnInfo, clientinfo = ClientInfo}) -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) - , proto_name => <<"COAP">> - , proto_ver => <<"1">> }, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, []]), diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index d79a880a1..925b7cc0e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -17,6 +17,7 @@ -module(emqx_gateway_api). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). -behaviour(minirest_api). @@ -243,7 +244,7 @@ schema_gateway_overview_list() -> %% %% NOTE: It is a temporary measure to generate swagger-schema -define(COAP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, @@ -331,7 +332,7 @@ schema_gateway_overview_list() -> ). -define(STOMP_GATEWAY_CONFS, -#{<<"authentication">> => +#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => #{<<"mechanism">> => <<"password-based">>, <<"name">> => <<"authenticator1">>, <<"server_type">> => <<"built-in-database">>, diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index ddbf99189..e97b0062d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -17,8 +17,6 @@ %% @doc The gateway configuration management module -module(emqx_gateway_conf). --include_lib("emqx/include/logger.hrl"). - %% Load/Unload -export([ load/0 , unload/0 @@ -56,6 +54,10 @@ , post_config_update/5 ]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). +-define(AUTHN_BIN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY). + -type atom_or_bin() :: atom() | binary(). -type ok_or_err() :: ok_or_err(). -type listener_ref() :: {ListenerType :: atom_or_bin(), @@ -106,8 +108,9 @@ maps_key_take([K | Ks], M, Acc) -> -spec update_gateway(atom_or_bin(), map()) -> ok_or_err(). update_gateway(GwName, Conf0) -> - Conf = maps:without([listeners, authentication, - <<"listeners">>, <<"authentication">>], Conf0), + Exclude0 = [listeners, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM], + Exclude1 = [atom_to_binary(K, utf8) || K <- Exclude0], + Conf = maps:without(Exclude0 ++ Exclude1, Conf0), update({?FUNCTION_NAME, bin(GwName), Conf}). %% FIXME: delete cert files ?? @@ -232,7 +235,7 @@ update(Req) -> res(emqx_conf:update([gateway], Req, #{override_to => cluster})). res({ok, _Result}) -> ok; -res({error, {error, {pre_config_update,emqx_gateway_conf,Reason}}}) -> {error, Reason}; +res({error, {pre_config_update, emqx_gateway_conf, Reason}}) -> {error, Reason}; res({error, Reason}) -> {error, Reason}. bin({LType, LName}) -> @@ -263,8 +266,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> undefined -> {error, not_found}; _ -> - NConf = maps:without([<<"listeners">>, - <<"authentication">>], Conf), + NConf = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), {ok, emqx_map_lib:deep_merge(RawConf, #{GwName => NConf})} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> @@ -311,11 +313,11 @@ pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})}; + #{GwName => #{?AUTHN_BIN => Conf}})}; _ -> {error, already_exist} end; @@ -326,9 +328,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> - NListener = maps:put(<<"authentication">>, Conf, Listener), + NListener = maps:put(?AUTHN_BIN, Conf, Listener), NGateway = #{GwName => #{<<"listeners">> => #{LType => #{LName => NListener}}}}, @@ -339,13 +341,13 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case emqx_map_lib:deep_get( - [GwName, <<"authentication">>], RawConf, undefined) of + [GwName, ?AUTHN_BIN], RawConf, undefined) of undefined -> {error, not_found}; _ -> {ok, emqx_map_lib:deep_merge( RawConf, - #{GwName => #{<<"authentication">> => Conf}})} + #{GwName => #{?AUTHN_BIN => Conf}})} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case emqx_map_lib:deep_get( @@ -354,12 +356,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> undefined -> {error, not_found}; Listener -> - case maps:get(<<"authentication">>, Listener, undefined) of + case maps:get(?AUTHN_BIN, Listener, undefined) of undefined -> {error, not_found}; Auth -> NListener = maps:put( - <<"authentication">>, + ?AUTHN_BIN, emqx_map_lib:deep_merge(Auth, Conf), Listener ), @@ -371,9 +373,9 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, emqx_map_lib:deep_remove( - [GwName, <<"authentication">>], RawConf)}; + [GwName, ?AUTHN_BIN], RawConf)}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> - Path = [GwName, <<"listeners">>, LType, LName, <<"authentication">>], + Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], {ok, emqx_map_lib:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 0d2f765c5..98344f968 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -19,6 +19,9 @@ -include("include/emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_authentication.hrl"). + +-define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). %% Mgmt APIs - gateway -export([ gateways/1 @@ -166,7 +169,7 @@ remove_listener(ListenerId) -> -spec authn(gateway_name()) -> map(). authn(GwName) -> %% XXX: Need append chain-nanme, authenticator-id? - Path = [gateway, GwName, authentication], + Path = [gateway, GwName, ?AUTHN], ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, @@ -176,7 +179,7 @@ authn(GwName) -> -spec authn(gateway_name(), binary()) -> map(). authn(GwName, ListenerId) -> {_, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), - Path = [gateway, GwName, listeners, Type, Name, authentication], + Path = [gateway, GwName, listeners, Type, Name, ?AUTHN], ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9f35225b7..5f87131d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -24,6 +24,7 @@ -dialyzer(no_unused). -dialyzer(no_fail_call). +-include_lib("emqx/include/emqx_authentication.hrl"). -include_lib("typerefl/include/types.hrl"). -type ip_port() :: tuple(). @@ -144,7 +145,7 @@ The client just sends its PUBLISH messages to a GW" , desc => "The Pre-defined topic ids and topic names.
A 'pre-defined' topic id is a topic id whose mapping to a topic name -is known in advance by both the client’s application and the gateway" +is known in advance by both the client's application and the gateway" })} , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); @@ -407,30 +408,14 @@ fields(dtls_opts) -> , ciphers => dtls_all_available }, false). -authentication() -> - sc(hoconsc:union( - [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]), - #{ nullable => {true, recursively} - , desc => +authentication_schema() -> + sc(emqx_authn_schema:authenticator_type(), + #{ nullable => {true, recursively} + , desc => """Default authentication configs for all of the gateway listeners.
For per-listener overrides see authentication in listener configs""" - }). + }). gateway_common_options() -> [ {enable, @@ -464,7 +449,7 @@ it has two purposes: sc(ref(clientinfo_override), #{ desc => "" })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} ]. common_listener_opts() -> @@ -483,7 +468,7 @@ common_listener_opts() -> sc(integer(), #{ default => 1000 })} - , {authentication, authentication()} + , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication_schema()} , {mountpoint, sc(binary(), #{ default => undefined diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index 0cd404154..423e38479 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -93,10 +93,10 @@ info(ctx, #channel{ctx = Ctx}) -> stats(_) -> []. -init(ConnInfo = #{peername := {PeerHost, _}, - sockname := {_, SockPort}}, +init(ConnInfoT = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> - Peercert = maps:get(peercert, ConnInfo, undefined), + Peercert = maps:get(peercert, ConnInfoT, undefined), Mountpoint = maps:get(mountpoint, Config, undefined), ListenerId = case maps:get(listener, Config, undefined) of undefined -> undefined; @@ -118,18 +118,20 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + ConnInfo = ConnInfoT#{proto_name => <<"LwM2M">>, proto_ver => <<"0.0">>}, + #channel{ ctx = Ctx , conninfo = ConnInfo , clientinfo = ClientInfo , timers = #{} , session = emqx_lwm2m_session:new() - %% FIXME: don't store anonymouse func + %% FIXME: don't store anonymouse func , with_context = with_context(Ctx, ClientInfo) }. with_context(Ctx, ClientInfo) -> fun(Type, Topic) -> - with_context(Type, Topic, Ctx, ClientInfo) + with_context(Type, Topic, Ctx, ClientInfo) end. lookup_cmd(Channel, Path, Action) -> @@ -293,7 +295,6 @@ check_lwm2m_version(#coap_message{options = Opts}, end, if IsValid -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) - , proto_name => <<"LwM2M">> , proto_ver => Ver }, {ok, Channel#channel{conninfo = NConnInfo}}; diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index e49a78e73..8e25a4e6e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -28,6 +28,8 @@ -include_lib("eunit/include/eunit.hrl"). +%% this parses to #{}, will not cause config cleanup +%% so we will need call emqx_config:erase -define(CONF_DEFAULT, <<" gateway {} ">>). @@ -39,6 +41,7 @@ gateway {} all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_config:erase(gateway), emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Conf. diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index f82b881a8..d45b5ad77 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -149,7 +149,7 @@ node_info(Node) when Node =:= node() -> Info#{node => node(), otp_release => iolist_to_binary(otp_rel()), memory_total => proplists:get_value(allocated, Memory), - memory_used => proplists:get_value(total, Memory), + memory_used => proplists:get_value(used, Memory), process_available => erlang:system_info(process_limit), process_used => erlang:system_info(process_count), @@ -650,4 +650,3 @@ max_row_limit() -> ?MAX_ROW_LIMIT. table_size(Tab) -> ets:info(Tab, size). - diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index 1605c3382..55c882f94 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -36,6 +36,7 @@ maybe_enable_modules() -> emqx_conf:get([telemetry, enable], true) andalso emqx_telemetry:enable(), emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:enable(), emqx_event_message:enable(), + emqx_conf_cli:load(), ok = emqx_rewrite:enable(), emqx_topic_metrics:enable(). @@ -45,4 +46,5 @@ maybe_disable_modules() -> emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:disable(), emqx_event_message:disable(), emqx_rewrite:disable(), + emqx_conf_cli:unload(), emqx_topic_metrics:disable(). diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 4d51834c2..27093ef1d 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -168,12 +168,11 @@ t_update_re_failed(_Config) -> }], Error = {badmatch, {error, - {error, - {emqx_modules_schema, - [{validation_error, - #{array_index => 1,path => "rewrite.re", - reason => {<<"*^test/*">>,{"nothing to repeat",0}}, - value => <<"*^test/*">>}}]}}}}, + {emqx_modules_schema, + [{validation_error, + #{path => "rewrite.1.re", + reason => {<<"*^test/*">>,{"nothing to repeat",0}}, + value => <<"*^test/*">>}}]}}}, ?assertError(Error, emqx_rewrite:update(Rules)), ok. diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_metrics.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_metrics.erl index 824890efc..d48b10dd1 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_metrics.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_metrics.erl @@ -27,7 +27,7 @@ -export([ inc/3 , inc/4 , get/3 - , get_speed/2 + , get_rate/2 , create_metrics/2 , clear_metrics/2 ]). @@ -54,7 +54,7 @@ -define(SECS_5M, 300). -define(SAMPLING, 10). -else. -%% Use 5 secs average speed instead of 5 mins in case of testing +%% Use 5 secs average rate instead of 5 mins in case of testing -define(SECS_5M, 5). -define(SAMPLING, 1). -endif. @@ -65,9 +65,9 @@ matched => integer(), success => integer(), failed => integer(), - speed => float(), - speed_max => float(), - speed_last5m => float() + rate => float(), + rate_max => float(), + rate_last5m => float() }. -type handler_name() :: atom(). -type metric_id() :: binary(). @@ -75,22 +75,22 @@ -define(CntrRef(Name), {?MODULE, Name}). -define(SAMPCOUNT_5M, (?SECS_5M div ?SAMPLING)). -%% the speed of 'matched' --record(speed, { +%% the rate of 'matched' +-record(rate, { max = 0 :: number(), current = 0 :: number(), last5m = 0 :: number(), - %% metadata for calculating the avg speed + %% metadata for calculating the avg rate tick = 1 :: number(), last_v = 0 :: number(), - %% metadata for calculating the 5min avg speed + %% metadata for calculating the 5min avg rate last5m_acc = 0 :: number(), last5m_smpl = [] :: list() }). -record(state, { metric_ids = sets:new(), - speeds :: undefined | #{metric_id() => #speed{}} + rates :: undefined | #{metric_id() => #rate{}} }). %%------------------------------------------------------------------------------ @@ -122,19 +122,19 @@ get(Name, Id, Metric) -> Ref -> counters:get(Ref, metrics_idx(Metric)) end. --spec(get_speed(handler_name(), metric_id()) -> map()). -get_speed(Name, Id) -> - gen_server:call(Name, {get_speed, Id}). +-spec(get_rate(handler_name(), metric_id()) -> map()). +get_rate(Name, Id) -> + gen_server:call(Name, {get_rate, Id}). -spec(get_metrics(handler_name(), metric_id()) -> metrics()). get_metrics(Name, Id) -> - #{max := Max, current := Current, last5m := Last5M} = get_speed(Name, Id), + #{max := Max, current := Current, last5m := Last5M} = get_rate(Name, Id), #{matched => get_matched(Name, Id), success => get_success(Name, Id), failed => get_failed(Name, Id), - speed => Current, - speed_max => Max, - speed_last5m => Last5M + rate => Current, + rate_max => Max, + rate_last5m => Last5M }. -spec inc(handler_name(), metric_id(), atom()) -> ok. @@ -176,35 +176,35 @@ start_link(Name) -> init(Name) -> erlang:process_flag(trap_exit, true), - %% the speed metrics + %% the rate metrics erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), persistent_term:put(?CntrRef(Name), #{}), {ok, #state{}}. -handle_call({get_speed, _Id}, _From, State = #state{speeds = undefined}) -> - {reply, format_speed(#speed{}), State}; -handle_call({get_speed, Id}, _From, State = #state{speeds = Speeds}) -> - {reply, case maps:get(Id, Speeds, undefined) of - undefined -> format_speed(#speed{}); - Speed -> format_speed(Speed) +handle_call({get_rate, _Id}, _From, State = #state{rates = undefined}) -> + {reply, format_rate(#rate{}), State}; +handle_call({get_rate, Id}, _From, State = #state{rates = Rates}) -> + {reply, case maps:get(Id, Rates, undefined) of + undefined -> format_rate(#rate{}); + Rate -> format_rate(Rate) end, State}; handle_call({create_metrics, Id}, _From, - State = #state{metric_ids = MIDs, speeds = Speeds}) -> + State = #state{metric_ids = MIDs, rates = Rates}) -> {reply, create_counters(get_self_name(), Id), State#state{metric_ids = sets:add_element(Id, MIDs), - speeds = case Speeds of - undefined -> #{Id => #speed{}}; - _ -> Speeds#{Id => #speed{}} + rates = case Rates of + undefined -> #{Id => #rate{}}; + _ -> Rates#{Id => #rate{}} end}}; handle_call({delete_metrics, Id}, _From, - State = #state{metric_ids = MIDs, speeds = Speeds}) -> + State = #state{metric_ids = MIDs, rates = Rates}) -> {reply, delete_counters(get_self_name(), Id), State#state{metric_ids = sets:del_element(Id, MIDs), - speeds = case Speeds of + rates = case Rates of undefined -> undefined; - _ -> maps:remove(Id, Speeds) + _ -> maps:remove(Id, Rates) end}}; handle_call(_Request, _From, State) -> @@ -213,17 +213,17 @@ handle_call(_Request, _From, State) -> handle_cast(_Msg, State) -> {noreply, State}. -handle_info(ticking, State = #state{speeds = undefined}) -> +handle_info(ticking, State = #state{rates = undefined}) -> erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), {noreply, State}; -handle_info(ticking, State = #state{speeds = Speeds0}) -> - Speeds = maps:map( - fun(Id, Speed) -> - calculate_speed(get_matched(get_self_name(), Id), Speed) - end, Speeds0), +handle_info(ticking, State = #state{rates = Rates0}) -> + Rates = maps:map( + fun(Id, Rate) -> + calculate_rate(get_matched(get_self_name(), Id), Rate) + end, Rates0), erlang:send_after(timer:seconds(?SAMPLING), self(), ticking), - {noreply, State#state{speeds = Speeds}}; + {noreply, State#state{rates = Rates}}; handle_info(_Info, State) -> {noreply, State}. @@ -261,38 +261,38 @@ get_couters_ref(Name, Id) -> get_all_counters(Name) -> persistent_term:get(?CntrRef(Name), #{}). -calculate_speed(_CurrVal, undefined) -> +calculate_rate(_CurrVal, undefined) -> undefined; -calculate_speed(CurrVal, #speed{max = MaxSpeed0, last_v = LastVal, - tick = Tick, last5m_acc = AccSpeed5Min0, +calculate_rate(CurrVal, #rate{max = MaxRate0, last_v = LastVal, + tick = Tick, last5m_acc = AccRate5Min0, last5m_smpl = Last5MinSamples0}) -> - %% calculate the current speed based on the last value of the counter - CurrSpeed = (CurrVal - LastVal) / ?SAMPLING, + %% calculate the current rate based on the last value of the counter + CurrRate = (CurrVal - LastVal) / ?SAMPLING, - %% calculate the max speed since the emqx startup - MaxSpeed = - if MaxSpeed0 >= CurrSpeed -> MaxSpeed0; - true -> CurrSpeed + %% calculate the max rate since the emqx startup + MaxRate = + if MaxRate0 >= CurrRate -> MaxRate0; + true -> CurrRate end, - %% calculate the average speed in last 5 mins + %% calculate the average rate in last 5 mins {Last5MinSamples, Acc5Min, Last5Min} = if Tick =< ?SAMPCOUNT_5M -> - Acc = AccSpeed5Min0 + CurrSpeed, - {lists:reverse([CurrSpeed | lists:reverse(Last5MinSamples0)]), + Acc = AccRate5Min0 + CurrRate, + {lists:reverse([CurrRate | lists:reverse(Last5MinSamples0)]), Acc, Acc / Tick}; true -> - [FirstSpeed | Speeds] = Last5MinSamples0, - Acc = AccSpeed5Min0 + CurrSpeed - FirstSpeed, - {lists:reverse([CurrSpeed | lists:reverse(Speeds)]), + [FirstRate | Rates] = Last5MinSamples0, + Acc = AccRate5Min0 + CurrRate - FirstRate, + {lists:reverse([CurrRate | lists:reverse(Rates)]), Acc, Acc / ?SAMPCOUNT_5M} end, - #speed{max = MaxSpeed, current = CurrSpeed, last5m = Last5Min, + #rate{max = MaxRate, current = CurrRate, last5m = Last5Min, last_v = CurrVal, last5m_acc = Acc5Min, last5m_smpl = Last5MinSamples, tick = Tick + 1}. -format_speed(#speed{max = Max, current = Current, last5m = Last5Min}) -> +format_rate(#rate{max = Max, current = Current, last5m = Last5Min}) -> #{max => Max, current => precision(Current, 2), last5m => precision(Last5Min, 2)}. precision(Float, N) -> diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_metrics_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_metrics_SUITE.erl index 3a74cd232..3f8a63f25 100644 --- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_metrics_SUITE.erl +++ b/apps/emqx_plugin_libs/test/emqx_plugin_libs_metrics_SUITE.erl @@ -24,7 +24,7 @@ all() -> [ {group, metrics} - , {group, speed} ]. + , {group, rate} ]. suite() -> [{ct_hooks, [cth_surefire]}, {timetrap, {seconds, 30}}]. @@ -34,8 +34,8 @@ groups() -> [ t_rule , t_no_creation_1 ]}, - {speed, [sequence], - [ rule_speed + {rate, [sequence], + [ rule_rate ]} ]. @@ -74,7 +74,7 @@ t_rule(_) -> ok = emqx_plugin_libs_metrics:clear_metrics(?NAME, <<"rule1">>), ok = emqx_plugin_libs_metrics:clear_metrics(?NAME, <<"rule2">>). -rule_speed(_) -> +rule_rate(_) -> ok = emqx_plugin_libs_metrics:create_metrics(?NAME, <<"rule1">>), ok = emqx_plugin_libs_metrics:create_metrics(?NAME, <<"rule:2">>), ok = emqx_plugin_libs_metrics:inc(?NAME, <<"rule1">>, 'rules.matched'), @@ -83,11 +83,11 @@ rule_speed(_) -> ?assertEqual(2, emqx_plugin_libs_metrics:get(?NAME, <<"rule1">>, 'rules.matched')), ct:sleep(1000), ?LET(#{max := Max, current := Current}, - emqx_plugin_libs_metrics:get_speed(?NAME, <<"rule1">>), + emqx_plugin_libs_metrics:get_rate(?NAME, <<"rule1">>), {?assert(Max =< 2), ?assert(Current =< 2)}), ct:sleep(2100), - ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_plugin_libs_metrics:get_speed(?NAME, <<"rule1">>), + ?LET(#{max := Max, current := Current, last5m := Last5Min}, emqx_plugin_libs_metrics:get_rate(?NAME, <<"rule1">>), {?assert(Max =< 2), ?assert(Current == 0), ?assert(Last5Min =< 0.67)}), diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index ff8cff158..9e8904a4e 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -52,7 +52,13 @@ prometheus_data_api() -> Metadata = #{ get => #{ description => <<"Get Prometheus Data">>, - responses => #{<<"200">> => schema(#{type => object})} + responses => #{<<"200">> => + #{content => + #{ + 'application/json' => #{schema => #{type => object}}, + 'text/plain' => #{schema => #{type => string}} + }} + } } }, {"/prometheus/stats", Metadata, stats}. @@ -72,8 +78,12 @@ prometheus(put, #{body := Body}) -> end, {200, emqx:get_raw_config([<<"prometheus">>], #{})}. -stats(get, #{query_string := Qs}) -> - Type = maps:get(<<"format_type">>, Qs, <<"json">>), +stats(get, #{headers := Headers}) -> + Type = + case maps:get(<<"accept">>, Headers, <<"text/plain">>) of + <<"application/json">> -> <<"json">>; + _ -> <<"prometheus">> + end, Data = emqx_prometheus:collect(Type), case Type of <<"json">> -> {200, Data}; diff --git a/apps/emqx_psk/etc/emqx_psk.conf b/apps/emqx_psk/etc/emqx_psk.conf index 80b29bfd4..ff9265fe1 100644 --- a/apps/emqx_psk/etc/emqx_psk.conf +++ b/apps/emqx_psk/etc/emqx_psk.conf @@ -2,11 +2,11 @@ ## EMQ X PSK ##-------------------------------------------------------------------- -psk { +psk_authentication { ## Whether to enable the PSK feature. enable = false - ## If init file is specified, emqx will import PSKs from the file + ## If init file is specified, emqx will import PSKs from the file ## into the built-in database at startup for use by the runtime. ## ## The file has to be structured line-by-line, each line must be in diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl index ff89041ce..085a533d7 100644 --- a/apps/emqx_psk/src/emqx_psk.erl +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -142,13 +142,13 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ get_config(enable) -> - emqx_conf:get([psk, enable]); + emqx_conf:get([psk_authentication, enable]); get_config(init_file) -> - emqx_conf:get([psk, init_file], undefined); + emqx_conf:get([psk_authentication, init_file], undefined); get_config(separator) -> - emqx_conf:get([psk, separator], ?DEFAULT_DELIMITER); + emqx_conf:get([psk_authentication, separator], ?DEFAULT_DELIMITER); get_config(chunk_size) -> - emqx_conf:get([psk, chunk_size]). + emqx_conf:get([psk_authentication, chunk_size]). import_psks(SrcFile) -> case file:open(SrcFile, [read, raw, binary, read_ahead]) of diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index cce51d3fa..8097ade94 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -24,9 +24,24 @@ , fields/1 ]). -roots() -> ["psk"]. +roots() -> ["psk_authentication"]. -fields("psk") -> +fields("psk_authentication") -> + #{fields => fields(), + desc => """PSK stands for 'Pre-Shared Keys'. +This config to enable TLS-PSK authentication. + +Important! Make sure the SSL listener with +only tlsv1.2 enabled, and also PSK cipher suites +configured, such as RSA-PSK-AES256-GCM-SHA384. +See listener SSL options config for more details. + +The IDs and secrets can be provided from a file the path +to which is configurable by the init_file field. +""" + }. + +fields() -> [ {enable, fun enable/1} , {init_file, fun init_file/1} , {separator, fun separator/1} @@ -43,7 +58,7 @@ init_file(desc) -> <<"If init_file is specified, emqx will import PSKs from the file ", "into the built-in database at startup for use by the runtime. ", "The file has to be structured line-by-line, each line must be in ", - "the format: :">>; + "the format of 'PSKIdentity:SharedSecret' for example: mydevice1:c2VjcmV0">>; init_file(nullable) -> true; init_file(_) -> undefined. diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index 5794b8634..36d9521fe 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -26,13 +26,13 @@ all() -> init_per_suite(Config) -> meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_config, get, fun([psk, enable]) -> true; - ([psk, chunk_size]) -> 50; + meck:expect(emqx_config, get, fun([psk_authentication, enable]) -> true; + ([psk_authentication, chunk_size]) -> 50; (KeyPath) -> meck:passthrough([KeyPath]) end), - meck:expect(emqx_config, get, fun([psk, init_file], _) -> + meck:expect(emqx_config, get, fun([psk_authentication, init_file], _) -> filename:join([code:lib_dir(emqx_psk, test), "data/init.psk"]); - ([psk, separator], _) -> <<":">>; + ([psk_authentication, separator], _) -> <<":">>; (KeyPath, Default) -> meck:passthrough([KeyPath, Default]) end), emqx_common_test_helpers:start_apps([emqx_psk]), diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index ba6bdfa6c..92dc62f24 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -29,6 +29,13 @@ emqx_retainer { ## Default: 0s msg_expiry_interval = 0s + ## When the retained flag of the PUBLISH message is set and Payload is empty, + ## whether to continue to publish the message. + ## see: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038 + ## + ## Default: false + #stop_publish_clear_msg = false + ## The message read and deliver flow rate control ## When a client subscribe to a wildcard topic, may many retained messages will be loaded. ## If you don't want these data loaded to the memory all at once, you can use this to control. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index 5d248e638..9be449b60 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -88,7 +88,12 @@ on_message_publish(Msg = #message{flags = #{retain := true}, payload = <<>>}, Context) -> delete_message(Context, Topic), - {ok, Msg}; + case get_stop_publish_clear_msg() of + true -> + {ok, emqx_message:set_header(allow_publish, false, Msg)}; + _ -> + {ok, Msg} + end; on_message_publish(Msg = #message{flags = #{retain := true}}, Context) -> Msg1 = emqx_message:set_header(retained, true, Msg), @@ -157,6 +162,9 @@ get_expiry_time(#message{timestamp = Ts}) -> _ -> Ts + Interval end. +get_stop_publish_clear_msg() -> + emqx_conf:get([?APP, stop_publish_clear_msg], false). + -spec update_config(hocon:config()) -> ok. update_config(Conf) -> gen_server:call(?MODULE, {?FUNCTION_NAME, Conf}). diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 55cfa2fcc..e1fa8373a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -14,6 +14,7 @@ fields("emqx_retainer") -> , {msg_clear_interval, sc(emqx_schema:duration_ms(), "0s")} , {flow_control, ?TYPE(hoconsc:ref(?MODULE, flow_control))} , {max_payload_size, sc(emqx_schema:bytesize(), "1MB")} + , {stop_publish_clear_msg, sc(boolean(), false)} , {config, config()} ]; diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index ccc647ddc..5596e9539 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -195,6 +195,21 @@ t_clean(_) -> ok = emqtt:disconnect(C1). +t_stop_publish_clear_msg(_) -> + emqx_retainer:update_config(#{<<"stop_publish_clear_msg">> => true}), + {ok, C1} = emqtt:start_link([{clean_start, true}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C1), + emqtt:publish(C1, <<"retained/0">>, <<"this is a retained message 0">>, [{qos, 0}, {retain, true}]), + + {ok, #{}, [0]} = emqtt:subscribe(C1, <<"retained/#">>, [{qos, 0}, {rh, 0}]), + ?assertEqual(1, length(receive_messages(1))), + + emqtt:publish(C1, <<"retained/0">>, <<"">>, [{qos, 0}, {retain, true}]), + ?assertEqual(0, length(receive_messages(1))), + + emqx_retainer:update_config(#{<<"stop_publish_clear_msg">> => false}), + ok = emqtt:disconnect(C1). + t_flow_control(_) -> emqx_retainer:update_config(#{<<"flow_control">> => #{<<"max_read_number">> => 1, <<"msg_deliver_quota">> => 1, diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 448f63138..1fe75447e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -32,13 +32,40 @@ check_params(Params, Tag) -> roots() -> [ {"rule_creation", sc(ref("rule_creation"), #{desc => "Schema for creating rules"})} + , {"rule_info", sc(ref("rule_info"), #{desc => "Schema for rule info"})} + , {"rule_events", sc(ref("rule_events"), #{desc => "Schema for rule events"})} , {"rule_test", sc(ref("rule_test"), #{desc => "Schema for testing rules"})} ]. fields("rule_creation") -> - [ {"id", sc(binary(), #{desc => "The Id of the rule", nullable => false})} + [ {"id", sc(binary(), + #{ desc => "The Id of the rule", nullable => false + , example => "my_rule_id" + })} ] ++ emqx_rule_engine_schema:fields("rules"); +fields("rule_info") -> + [ {"metrics", sc(ref("metrics"), #{desc => "The metrics of the rule"})} + , {"node_metrics", sc(ref("node_metrics"), #{desc => "The metrics of the rule"})} + , {"from", sc(hoconsc:array(binary()), + #{desc => "The topics of the rule", example => "t/#"})} + , {"created_at", sc(binary(), + #{ desc => "The created time of the rule" + , example => "2021-12-01T15:00:43.153+08:00" + })} + ] ++ fields("rule_creation"); + +%% TODO: we can delete this API if the Dashboard not denpends on it +fields("rule_events") -> + ETopics = [emqx_rule_events:event_topic(E) || E <- emqx_rule_events:event_names()], + [ {"event", sc(hoconsc:enum(ETopics), #{desc => "The event topics", nullable => false})} + , {"title", sc(binary(), #{desc => "The title", example => "some title"})} + , {"description", sc(binary(), #{desc => "The description", example => "some desc"})} + , {"columns", sc(map(), #{desc => "The columns"})} + , {"test_columns", sc(map(), #{desc => "The test columns"})} + , {"sql_example", sc(binary(), #{desc => "The sql_example"})} + ]; + fields("rule_test") -> [ {"context", sc(hoconsc:union([ ref("ctx_pub") , ref("ctx_sub") @@ -53,6 +80,18 @@ fields("rule_test") -> , {"sql", sc(binary(), #{desc => "The SQL of the rule for testing", nullable => false})} ]; +fields("metrics") -> + [ {"matched", sc(integer(), #{desc => "How much times this rule is matched"})} + , {"rate", sc(float(), #{desc => "The rate of matched, times/second"})} + , {"rate_max", sc(float(), #{desc => "The max rate of matched, times/second"})} + , {"rate_last5m", sc(float(), + #{desc => "The average rate of matched in last 5 mins, times/second"})} + ]; + +fields("node_metrics") -> + [ {"node", sc(binary(), #{desc => "The node name", example => "emqx@127.0.0.1"})} + ] ++ fields("metrics"); + fields("ctx_pub") -> [ {"event_type", sc(message_publish, #{desc => "Event Type", nullable => false})} , {"id", sc(binary(), #{desc => "Message ID"})} diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 9e341b388..75238fb71 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -18,16 +18,17 @@ -include("rule_engine.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). -behaviour(minirest_api). --export([api_spec/0]). +-import(hoconsc, [mk/2, ref/2, array/1]). --export([ crud_rules/2 - , list_events/2 - , crud_rules_by_id/2 - , rule_test/2 - ]). +%% Swagger specs from hocon schema +-export([api_spec/0, paths/0, schema/1, namespace/0]). + +%% API callbacks +-export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2]). -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))). -define(ERR_BADARGS(REASON), @@ -43,210 +44,130 @@ {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(REASON)}} end). +namespace() -> "rule". + api_spec() -> - { - [ api_rules_list_create() - , api_rules_crud() - , api_rule_test() - , api_events_list() - ], - [] - }. + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). -api_rules_list_create() -> - Metadata = #{ +paths() -> ["/rule_events", "/rule_test", "/rules", "/rules/:id"]. + +error_schema(Code, Message) -> + [ {code, mk(string(), #{example => Code})} + , {message, mk(string(), #{example => Message})} + ]. + +rule_creation_schema() -> + ref(emqx_rule_api_schema, "rule_creation"). + +rule_update_schema() -> + ref(emqx_rule_engine_schema, "rules"). + +rule_test_schema() -> + ref(emqx_rule_api_schema, "rule_test"). + +rule_info_schema() -> + ref(emqx_rule_api_schema, "rule_info"). + +schema("/rules") -> + #{ + operationId => '/rules', get => #{ + tags => [<<"rules">>], description => <<"List all rules">>, + summary => <<"List Rules">>, responses => #{ - <<"200">> => - emqx_mgmt_util:array_schema(resp_schema(), <<"List rules successfully">>)}}, + 200 => mk(array(rule_info_schema()), #{desc => "List of rules"}) + }}, post => #{ - description => <<"Create a new rule using given Id to all nodes in the cluster">>, - 'requestBody' => emqx_mgmt_util:schema(post_req_schema(), <<"Rule parameters">>), + tags => [<<"rules">>], + description => <<"Create a new rule using given Id">>, + summary => <<"Create a Rule">>, + requestBody => rule_creation_schema(), responses => #{ - <<"400">> => - emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), - <<"201">> => - emqx_mgmt_util:schema(resp_schema(), <<"Create rule successfully">>)}} - }, - {"/rules", Metadata, crud_rules}. + 400 => error_schema('BAD_ARGS', "Invalid Parameters"), + 201 => rule_info_schema() + }} + }; -api_events_list() -> - Metadata = #{ +schema("/rule_events") -> + #{ + operationId => '/rule_events', get => #{ + tags => [<<"rules">>], description => <<"List all events can be used in rules">>, + summary => <<"List Events">>, responses => #{ - <<"200">> => - emqx_mgmt_util:array_schema(resp_schema(), <<"List events successfully">>)}} - }, - {"/rule_events", Metadata, list_events}. + 200 => mk(ref(emqx_rule_api_schema, "rule_events"), #{}) + } + } + }; -api_rules_crud() -> - Metadata = #{ +schema("/rules/:id") -> + #{ + operationId => '/rules/:id', get => #{ + tags => [<<"rules">>], description => <<"Get a rule by given Id">>, - parameters => [param_path_id()], + summary => <<"Get a Rule">>, + parameters => param_path_id(), responses => #{ - <<"404">> => - emqx_mgmt_util:error_schema(<<"Rule not found">>, ['NOT_FOUND']), - <<"200">> => - emqx_mgmt_util:schema(resp_schema(), <<"Get rule successfully">>)}}, + 404 => error_schema('NOT_FOUND', "Rule not found"), + 200 => rule_info_schema() + } + }, put => #{ - description => <<"Create or update a rule by given Id to all nodes in the cluster">>, - parameters => [param_path_id()], - 'requestBody' => emqx_mgmt_util:schema(put_req_schema(), <<"Rule parameters">>), + tags => [<<"rules">>], + description => <<"Update a rule by given Id to all nodes in the cluster">>, + summary => <<"Update a Rule">>, + parameters => param_path_id(), + requestBody => rule_update_schema(), responses => #{ - <<"400">> => - emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), - <<"200">> => - emqx_mgmt_util:schema(resp_schema(), - <<"Create or update rule successfully">>)}}, + 400 => error_schema('BAD_ARGS', "Invalid Parameters"), + 200 => rule_info_schema() + } + }, delete => #{ + tags => [<<"rules">>], description => <<"Delete a rule by given Id from all nodes in the cluster">>, - parameters => [param_path_id()], + summary => <<"Delete a Rule">>, + parameters => param_path_id(), responses => #{ - <<"204">> => - emqx_mgmt_util:schema(<<"Delete rule successfully">>)}} - }, - {"/rules/:id", Metadata, crud_rules_by_id}. + 204 => <<"Delete rule successfully">> + } + } + }; -api_rule_test() -> - Metadata = #{ +schema("/rule_test") -> + #{ + operationId => '/rule_test', post => #{ + tags => [<<"rules">>], description => <<"Test a rule">>, - 'requestBody' => emqx_mgmt_util:schema(rule_test_req_schema(), <<"Rule parameters">>), + summary => <<"Test a Rule">>, + requestBody => rule_test_schema(), responses => #{ - <<"400">> => - emqx_mgmt_util:error_schema(<<"Invalid Parameters">>, ['BAD_ARGS']), - <<"412">> => - emqx_mgmt_util:error_schema(<<"SQL Not Match">>, ['NOT_MATCH']), - <<"200">> => - emqx_mgmt_util:schema(rule_test_resp_schema(), <<"Rule Test Pass">>)}} - }, - {"/rule_test", Metadata, rule_test}. - -put_req_schema() -> - #{type => object, - properties => #{ - sql => #{ - description => <<"The SQL">>, - type => string, - example => <<"SELECT * from \"t/1\"">> - }, - enable => #{ - description => <<"Enable or disable the rule">>, - type => boolean, - example => true - }, - outputs => #{ - description => <<"The outputs of the rule">>, - type => array, - items => #{ - 'oneOf' => [ - #{ - type => string, - example => <<"channel_id_of_my_bridge">>, - description => <<"The channel id of an emqx bridge">> - }, - #{ - type => object, - properties => #{ - function => #{ - type => string, - example => <<"console">> - } - } - } - ] + 400 => error_schema('BAD_ARGS', "Invalid Parameters"), + 412 => error_schema('NOT_MATCH', "SQL Not Match"), + 200 => <<"Rule Test Pass">> } - }, - description => #{ - description => <<"The description for the rule">>, - type => string, - example => <<"A simple rule that handles MQTT messages from topic \"t/1\"">> } - } }. -post_req_schema() -> - Req = #{properties := Prop} = put_req_schema(), - Req#{properties => Prop#{ - id => #{ - description => <<"The Id for the rule">>, - example => <<"my_rule">>, - type => string - } - }}. - -resp_schema() -> - Req = #{properties := Prop} = put_req_schema(), - Req#{properties => Prop#{ - id => #{ - description => <<"The Id for the rule">>, - type => string - }, - created_at => #{ - description => <<"The time that this rule was created, in rfc3339 format">>, - type => string, - example => <<"2021-09-18T13:57:29+08:00">> - } - }}. - -rule_test_req_schema() -> - #{type => object, properties => #{ - sql => #{ - description => <<"The SQL">>, - type => string, - example => <<"SELECT * from \"t/1\"">> - }, - context => #{ - type => object, - properties => #{ - event_type => #{ - description => <<"Event Type">>, - type => string, - enum => [<<"message_publish">>, <<"message_acked">>, <<"message_delivered">>, - <<"message_dropped">>, <<"session_subscribed">>, <<"session_unsubscribed">>, - <<"client_connected">>, <<"client_disconnected">>], - example => <<"message_publish">> - }, - clientid => #{ - description => <<"The Client ID">>, - type => string, - example => <<"\"c_emqx\"">> - }, - topic => #{ - description => <<"The Topic">>, - type => string, - example => <<"t/1">> - } - } - } - }}. - -rule_test_resp_schema() -> - #{type => object}. - param_path_id() -> - #{ - name => id, - in => path, - schema => #{type => string}, - required => true - }. + [{id, mk(binary(), #{in => path, example => <<"my_rule_id">>})}]. %%------------------------------------------------------------------------------ %% Rules API %%------------------------------------------------------------------------------ -list_events(#{}, _Params) -> +'/rule_events'(get, _Params) -> {200, emqx_rule_events:event_info()}. -crud_rules(get, _Params) -> +'/rules'(get, _Params) -> Records = emqx_rule_engine:get_rules_ordered_by_ts(), {200, format_rule_resp(Records)}; -crud_rules(post, #{body := #{<<"id">> := Id} = Params}) -> +'/rules'(post, #{body := #{<<"id">> := Id} = Params}) -> ConfPath = emqx_rule_engine:config_key_path() ++ [Id], case emqx_rule_engine:get_rule(Id) of {ok, _Rule} -> @@ -263,13 +184,13 @@ crud_rules(post, #{body := #{<<"id">> := Id} = Params}) -> end end. -rule_test(post, #{body := Params}) -> +'/rule_test'(post, #{body := Params}) -> ?CHECK_PARAMS(Params, rule_test, case emqx_rule_sqltester:test(CheckedParams) of {ok, Result} -> {200, Result}; {error, nomatch} -> {412, #{code => 'NOT_MATCH', message => <<"SQL Not Match">>}} end). -crud_rules_by_id(get, #{bindings := #{id := Id}}) -> +'/rules/:id'(get, #{bindings := #{id := Id}}) -> case emqx_rule_engine:get_rule(Id) of {ok, Rule} -> {200, format_rule_resp(Rule)}; @@ -277,7 +198,7 @@ crud_rules_by_id(get, #{bindings := #{id := Id}}) -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end; -crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params}) -> +'/rules/:id'(put, #{bindings := #{id := Id}, body := Params}) -> ConfPath = emqx_rule_engine:config_key_path() ++ [Id], case emqx:update_config(ConfPath, maps:remove(<<"id">>, Params), #{}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> @@ -289,7 +210,7 @@ crud_rules_by_id(put, #{bindings := #{id := Id}, body := Params}) -> {400, #{code => 'BAD_ARGS', message => ?ERR_BADARGS(Reason)}} end; -crud_rules_by_id(delete, #{bindings := #{id := Id}}) -> +'/rules/:id'(delete, #{bindings := #{id := Id}}) -> ConfPath = emqx_rule_engine:config_key_path() ++ [Id], case emqx:remove_config(ConfPath, #{}) of {ok, _} -> {204}; @@ -315,11 +236,13 @@ format_rule_resp(#{ id := Id, created_at := CreatedAt, sql := SQL, enabled := Enabled, description := Descr}) -> + NodeMetrics = get_rule_metrics(Id), #{id => Id, from => Topics, outputs => format_output(Output), sql => SQL, - metrics => get_rule_metrics(Id), + metrics => aggregate_metrics(NodeMetrics), + node_metrics => NodeMetrics, enabled => Enabled, created_at => format_datetime(CreatedAt, millisecond), description => Descr @@ -339,19 +262,28 @@ do_format_output(BridgeChannelId) when is_binary(BridgeChannelId) -> get_rule_metrics(Id) -> Format = fun (Node, #{matched := Matched, - speed := Current, - speed_max := Max, - speed_last5m := Last5M + rate := Current, + rate_max := Max, + rate_last5m := Last5M }) -> #{ matched => Matched - , speed => Current - , speed_max => Max - , speed_last5m => Last5M + , rate => Current + , rate_max => Max + , rate_last5m => Last5M , node => Node } end, [Format(Node, rpc:call(Node, emqx_plugin_libs_metrics, get_metrics, [rule_metrics, Id])) || Node <- mria_mnesia:running_nodes()]. +aggregate_metrics(AllMetrics) -> + InitMetrics = #{matched => 0, rate => 0, rate_max => 0, rate_last5m => 0}, + lists:foldl(fun + (#{matched := Match1, rate := Rate1, rate_max := RateMax1, rate_last5m := Rate5m1}, + #{matched := Match0, rate := Rate0, rate_max := RateMax0, rate_last5m := Rate5m0}) -> + #{matched => Match1 + Match0, rate => Rate1 + Rate0, + rate_max => RateMax1 + RateMax0, rate_last5m => Rate5m1 + Rate5m0} + end, InitMetrics, AllMetrics). + get_one_rule(AllRules, Id) -> [R || R = #{id := Id0} <- AllRules, Id0 == Id]. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 8daae99b5..93661ab53 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -44,19 +44,17 @@ fields("rules") -> SQL query to transform the messages.
Example: SELECT * FROM \"test/topic\" WHERE payload.x = 1
""" + , example => "SELECT * FROM \"test/topic\" WHERE payload.x = 1" , nullable => false - , validator => fun ?MODULE:validate_sql/1})} - , {"outputs", sc(hoconsc:array(hoconsc:union( - [ binary() - , ref("builtin_output_republish") - , ref("builtin_output_console") - ])), + , validator => fun ?MODULE:validate_sql/1 + })} + , {"outputs", sc(hoconsc:array(hoconsc:union(outputs())), #{ desc => """ A list of outputs of the rule.
An output can be a string that refers to the channel Id of a emqx bridge, or a object that refers to a function.
There a some built-in functions like \"republish\" and \"console\", and we also support user -provided functions like \":\".
+provided functions in the format: \"{module}:{function}\".
The outputs in the list is executed one by one in order. This means that if one of the output is executing slowly, all of the outputs comes after it will not be executed until it returns.
@@ -66,9 +64,19 @@ If there's any error when running an output, there will be an error message, and counter of the function output or the bridge channel will increase. """ , default => [] + , example => [ + <<"http:my_http_bridge">>, + #{function => republish, args => #{ + topic => <<"t/1">>, payload => <<"${payload}">>}}, + #{function => console} + ] })} , {"enable", sc(boolean(), #{desc => "Enable or disable the rule", default => true})} - , {"description", sc(binary(), #{desc => "The description of the rule", default => <<>>})} + , {"description", sc(binary(), + #{ desc => "The description of the rule" + , example => "Some description" + , default => <<>> + })} ]; fields("builtin_output_republish") -> @@ -106,6 +114,27 @@ fields("builtin_output_console") -> % default => #{}})} ]; +fields("user_provided_function") -> + [ {function, sc(binary(), + #{ desc => """ +The user provided function. Should be in the format: '{module}:{function}'.
+Where the is the erlang callback module and the {function} is the erlang function.
+To write your own function, checkout the function console and +republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_outputs.erl as an example. +""" + , example => "module:function" + })} + , {args, sc(map(), + #{ desc => """ +The args will be passed as the 3rd argument to module:function/3, +checkout the function console and republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_outputs.erl as an example. +""" + , default => #{} + })} + ]; + fields("republish_args") -> [ {topic, sc(binary(), #{ desc =>""" @@ -113,8 +142,9 @@ The target topic of message to be re-published.
Template with variables is allowed, see description of the 'republish_args'. """ , nullable => false + , example => <<"a/1">> })} - , {qos, sc(binary(), + , {qos, sc(qos(), #{ desc => """ The qos of the message to be re-published. Template with with variables is allowed, see description of the 'republish_args.
@@ -122,8 +152,9 @@ Defaults to ${qos}. If variable ${qos} is not found from the selected result of 0 is used. """ , default => <<"${qos}">> + , example => <<"${qos}">> })} - , {retain, sc(binary(), + , {retain, sc(hoconsc:union([binary(), boolean()]), #{ desc => """ The retain flag of the message to be re-published. Template with with variables is allowed, see description of the 'republish_args.
@@ -131,6 +162,7 @@ Defaults to ${retain}. If variable ${retain} is not found from the selected resu of the rule, false is used. """ , default => <<"${retain}">> + , example => <<"${retain}">> })} , {payload, sc(binary(), #{ desc => """ @@ -140,9 +172,20 @@ Defaults to ${payload}. If variable ${payload} is not found from the selected re of the rule, then the string \"undefined\" is used. """ , default => <<"${payload}">> + , example => <<"${payload}">> })} ]. +outputs() -> + [ binary() + , ref("builtin_output_republish") + , ref("builtin_output_console") + , ref("user_provided_function") + ]. + +qos() -> + hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2), binary()]). + validate_sql(Sql) -> case emqx_rule_sqlparser:parse(Sql) of {ok, _Result} -> ok; diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 0aff9f018..c61629b39 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -25,7 +25,9 @@ , load/1 , unload/0 , unload/1 + , event_names/0 , event_name/1 + , event_topic/1 , eventmsg_publish/1 ]). @@ -45,17 +47,6 @@ , columns_with_exam/1 ]). --define(SUPPORTED_HOOK, - [ 'client.connected' - , 'client.disconnected' - , 'session.subscribed' - , 'session.unsubscribed' - , 'message.publish' - , 'message.delivered' - , 'message.acked' - , 'message.dropped' - ]). - -ifdef(TEST). -export([ reason/1 , hook_fun/1 @@ -63,6 +54,17 @@ ]). -endif. +event_names() -> + [ 'client.connected' + , 'client.disconnected' + , 'session.subscribed' + , 'session.unsubscribed' + , 'message.publish' + , 'message.delivered' + , 'message.acked' + , 'message.dropped' + ]. + reload() -> lists:foreach(fun(Rule) -> ok = emqx_rule_engine:load_hooks_for_rule(Rule) @@ -78,7 +80,7 @@ load(Topic) -> unload() -> lists:foreach(fun(HookPoint) -> emqx_hooks:del(HookPoint, {?MODULE, hook_fun(HookPoint)}) - end, ?SUPPORTED_HOOK). + end, event_names()). unload(Topic) -> HookPoint = event_name(Topic), diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 7b68b3ee3..1cabf3e32 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -247,9 +247,9 @@ handle_output(OutId, Selected, Envs) -> }) end. -do_handle_output(ChannelId, Selected, _Envs) when is_binary(ChannelId) -> - ?SLOG(debug, #{msg => "output to bridge", channel_id => ChannelId}), - emqx_bridge:send_message(ChannelId, Selected); +do_handle_output(BridgeId, Selected, _Envs) when is_binary(BridgeId) -> + ?SLOG(debug, #{msg => "output to bridge", bridge_id => BridgeId}), + emqx_bridge:send_message(BridgeId, Selected); do_handle_output(#{mod := Mod, func := Func, args := Args}, Selected, Envs) -> Mod:Func(Selected, Envs, Args). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 712d113f9..4dd564b36 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -36,34 +36,34 @@ t_crud_rule_api(_Config) -> <<"outputs">> => [#{<<"function">> => <<"console">>}], <<"sql">> => <<"SELECT * from \"t/1\"">> }, - {201, Rule} = emqx_rule_engine_api:crud_rules(post, #{body => Params0}), + {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}), %% if we post again with the same params, it return with 400 "rule id already exists" ?assertMatch({400, #{code := _, message := _Message}}, - emqx_rule_engine_api:crud_rules(post, #{body => Params0})), + emqx_rule_engine_api:'/rules'(post, #{body => Params0})), ?assertEqual(RuleID, maps:get(id, Rule)), - {200, Rules} = emqx_rule_engine_api:crud_rules(get, #{}), + {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), - {200, Rule1} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), ct:pal("RShow : ~p", [Rule1]), ?assertEqual(Rule, Rule1), - {200, Rule2} = emqx_rule_engine_api:crud_rules_by_id(put, #{ + {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ bindings => #{id => RuleID}, body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} }), - {200, Rule3} = emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}}), + {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), %ct:pal("RShow : ~p", [Rule3]), ?assertEqual(Rule3, Rule2), ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), - ?assertMatch({204}, emqx_rule_engine_api:crud_rules_by_id(delete, + ?assertMatch({204}, emqx_rule_engine_api:'/rules/:id'(delete, #{bindings => #{id => RuleID}})), %ct:pal("Show After Deleted: ~p", [NotFound]), ?assertMatch({404, #{code := _, message := _Message}}, - emqx_rule_engine_api:crud_rules_by_id(get, #{bindings => #{id => RuleID}})), + emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}})), ok. diff --git a/bin/emqx b/bin/emqx index e1b72d1f9..7862a98b8 100755 --- a/bin/emqx +++ b/bin/emqx @@ -342,6 +342,9 @@ generate_config() { NOW_TIME="$(call_hocon now_time)" ## ths command populates two files: app.