diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 6b0c6e722..a21f130d4 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -198,9 +198,6 @@ jobs: fail-fast: false matrix: app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} - profile: - - emqx - - emqx-enterprise runs-on: - aws-amd64 - ubuntu-20.04 diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index f17a8d3f2..9f03ee307 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.11"). +-define(EMQX_RELEASE_CE, "5.0.12"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.0-beta.5"). +-define(EMQX_RELEASE_EE, "5.0.0-beta.6"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d91b32da0..ff27566ec 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index fa1c63868..16d9f31a7 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -402,6 +402,7 @@ merge_envs(SchemaMod, RawConf) -> required => false, format => map, apply_override_envs => true, + remove_env_meta => true, check_lazy => true }, hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts). @@ -575,10 +576,10 @@ load_hocon_file(FileName, LoadType) -> end. do_get_raw(Path) -> - hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path)). + do_get(?RAW_CONF, Path). do_get_raw(Path, Default) -> - hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path, Default)). + do_get(?RAW_CONF, Path, Default). do_get(Type, KeyPath) -> Ref = make_ref(), diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index ac0ee0eb0..c7f9d6fdf 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -39,12 +39,15 @@ providers() -> {{scram, built_in_database}, emqx_enhanced_authn_scram_mnesia} ]. -check_configs(C) when is_map(C) -> - check_configs([C]); -check_configs([]) -> +check_configs(CM) when is_map(CM) -> + check_configs([CM]); +check_configs(CL) -> + check_configs(CL, 1). + +check_configs([], _Nth) -> []; -check_configs([Config | Configs]) -> - [check_config(Config) | check_configs(Configs)]. +check_configs([Config | Configs], Nth) -> + [check_config(Config, #{id_for_log => Nth}) | check_configs(Configs, Nth + 1)]. check_config(Config) -> check_config(Config, #{}). @@ -55,15 +58,16 @@ check_config(Config, Opts) -> #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults end. -do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> +do_check_config(#{<<"mechanism">> := Mec0} = Config, Opts) -> + Mec = atom(Mec0, #{error => unknown_mechanism}), Key = case maps:get(<<"backend">>, Config, false) of - false -> atom(Mec); - Backend -> {atom(Mec), atom(Backend)} + false -> Mec; + Backend -> {Mec, atom(Backend, #{error => unknown_backend})} end, case lists:keyfind(Key, 1, providers()) of false -> - throw({unknown_handler, Key}); + throw(#{error => unknown_authn_provider, which => Key}); {_, ProviderModule} -> hocon_tconf:check_plain( ProviderModule, @@ -71,22 +75,22 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) -> Opts#{atom_key => true} ) end; -do_check_config(Config, _Opts) when is_map(Config) -> - throw({invalid_config, "mechanism_field_required", Config}); -do_check_config(RawConf, Opts) -> - %% authentication conf is lazy type, when it comes from ENV, it is a string - %% EMQX_AUTHENTICATION__1="{mechanism=\"password_based\"...}" - case hocon:binary(RawConf, Opts) of - {ok, Conf} -> do_check_config(Conf, Opts); - {error, Reason} -> throw({invalid_config, Reason}) - end. +do_check_config(Config, Opts) when is_map(Config) -> + throw(#{ + error => invalid_config, + which => maps:get(id_for_log, Opts, unknown), + reason => "mechanism_field_required" + }). -atom(Bin) -> +%% The atoms have to be loaded already, +%% which might be an issue for plugins which are loaded after node boot +%% but they should really manage their own configs in that case. +atom(Bin, ErrorContext) -> try binary_to_existing_atom(Bin, utf8) catch _:_ -> - throw({unknown_auth_provider, Bin}) + throw(ErrorContext#{value => Bin}) end. -spec get_enabled_authns() -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index ad35485ed..ef5dd6093 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -298,8 +298,8 @@ parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) -> %% For some drivers that can be used as data-sources, we need to provide a %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it %% receives a message from the external database. - BName = bridge_id(Type, Name), - Conf#{hookpoint => <<"$bridges/", BName/binary>>, bridge_name => Name}; + BId = bridge_id(Type, Name), + Conf#{hookpoint => <<"$bridges/", BId/binary>>, bridge_name => Name}; parse_confs(_Type, _Name, Conf) -> Conf. diff --git a/bin/emqx b/bin/emqx index 199898ea4..f4c52d1e4 100755 --- a/bin/emqx +++ b/bin/emqx @@ -115,14 +115,14 @@ usage() { echo "Print path to Erlang runtime bin dir" ;; rpc) - echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]" + echo "Usage: $REL_NAME rpc MODULE FUNCTION [ARGS, ...]" echo "Connect to the EMQX node and make an Erlang RPC" echo "This command blocks for at most 60 seconds." echo "It exits with non-zero code in case of any RPC failure" echo "including connection error and runtime exception" ;; rpcterms) - echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]" + echo "Usage: $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]" echo "Connect to the EMQX node and make an Erlang RPC" echo "The result of the RPC call is pretty-printed as an " echo "Erlang term" diff --git a/changes/v5.0.12-en.md b/changes/v5.0.12-en.md index 51b1bb9de..c99b1bb95 100644 --- a/changes/v5.0.12-en.md +++ b/changes/v5.0.12-en.md @@ -36,7 +36,7 @@ Please note, the request body of `/bridges` API to configure MQTT brdige is chan - Return `204` instead of `200` for `POST /gateway/lwm2m/clients/{clientid}/{read,write,observe}` [#9480](https://github.com/emqx/emqx/pull/9480). -- Make possible to create an authentication entirely from environment variable [#9437](https://github.com/emqx/emqx/pull/9437). +- Make possible to create an authentication entirely from environment variable [#9547](https://github.com/emqx/emqx/pull/9547). As an example, one can now enable MySQL auth with: `env EMQX_AUTHENTICATION__1='{mechanism="password_based",backend="mysql",server="localhost:3306",database="emqx",username="emqx",password="******",query="SELECT password_hash,salt FROM mqtt_user WHERE username=${username} LIMIT 1",enable=true}'`. Prior to this change, overrides only work on top of existing authentication, for example, if there is already MySQL auth configured in `emqx.conf` diff --git a/changes/v5.0.12-zh.md b/changes/v5.0.12-zh.md index 2cf1a6476..6d8ed4643 100644 --- a/changes/v5.0.12-zh.md +++ b/changes/v5.0.12-zh.md @@ -35,7 +35,7 @@ v5.0.11 或更早版本创建的配置文件,在新版本中会被自动转换 - 现在调用 `POST /gateway/lwm2m/clients/{clientid}/{read,write,observe}` 时,将会返回 204,而不再是 200 [#9480](https://github.com/emqx/emqx/pull/9480)。 -- 允许使用环境变量来创建一个认证配置 [#9437](https://github.com/emqx/emqx/pull/9437)。 +- 允许使用环境变量来创建一个认证配置 [#9547](https://github.com/emqx/emqx/pull/9547)。 例如,现在可以用如下环境变量来创建一个 MySQL 认证: `env EMQX_AUTHENTICATION__1='{mechanism="password_based",backend="mysql",server="localhost:3306",database="emqx",username="emqx",password="******",query="SELECT password_hash,salt FROM mqtt_user WHERE username=${username} LIMIT 1",enable=true}'`。 在此之前,环境变量的重载仅作用于已经存在的配置之上,例如,当 `emqx.conf` 中已经配置了一个 MySQL 认证,那么可以使用如下方法来将它禁用: diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index f3f33c984..8bf0ee0e3 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.11 +version: 5.0.12 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.11 +appVersion: 5.0.12 diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 9119b052d..ee112def8 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,5 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}} +{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}} , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} diff --git a/mix.exs b/mix.exs index 7aa632bd9..11ef14eb1 100644 --- a/mix.exs +++ b/mix.exs @@ -67,7 +67,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.31.2", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 3e1eafc0f..e9678b1cd 100644 --- a/rebar.config +++ b/rebar.config @@ -67,7 +67,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 512abdab6..bf0e98229 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -14,7 +14,11 @@ case "$UNAME" in SYSTEM="${DIST}${VERSION_ID}" ;; Linux) - if grep -q -i 'rhel' /etc/*-release; then + # /etc/os-release on amazon linux 2 contains both rhel and centos strings + if grep -q -i 'amzn' /etc/*-release; then + DIST='amzn' + VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" + elif grep -q -i 'rhel' /etc/*-release; then DIST='el' VERSION_ID="$(rpm --eval '%{rhel}')" else