Merge remote-tracking branch 'origin/release-50' into 1214-sync-master-upstreams

.github/workflows/run_test_cases.yaml

@@ -198,9 +198,6 @@ jobs:
           fail-fast: false
           matrix:
             app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
-            profile:
-              - emqx
-              - emqx-enterprise
             runs-on:
               - aws-amd64
               - ubuntu-20.04

apps/emqx/include/emqx_release.hrl

@@ -32,10 +32,10 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.11").
+-define(EMQX_RELEASE_CE, "5.0.12").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.0-beta.5").
+-define(EMQX_RELEASE_EE, "5.0.0-beta.6").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/rebar.config

@@ -29,7 +29,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}

apps/emqx/src/emqx_config.erl

@@ -402,6 +402,7 @@ merge_envs(SchemaMod, RawConf) ->
         required => false,
         format => map,
         apply_override_envs => true,
+        remove_env_meta => true,
         check_lazy => true
     },
     hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).
@@ -575,10 +576,10 @@ load_hocon_file(FileName, LoadType) ->
     end.
 
 do_get_raw(Path) ->
-    hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path)).
+    do_get(?RAW_CONF, Path).
 
 do_get_raw(Path, Default) ->
-    hocon_tconf:remove_env_meta(do_get(?RAW_CONF, Path, Default)).
+    do_get(?RAW_CONF, Path, Default).
 
 do_get(Type, KeyPath) ->
     Ref = make_ref(),

apps/emqx_authn/src/emqx_authn.erl

@@ -39,12 +39,15 @@ providers() ->
         {{scram, built_in_database}, emqx_enhanced_authn_scram_mnesia}
     ].
 
-check_configs(C) when is_map(C) ->
-    check_configs([C]);
-check_configs([]) ->
+check_configs(CM) when is_map(CM) ->
+    check_configs([CM]);
+check_configs(CL) ->
+    check_configs(CL, 1).
+
+check_configs([], _Nth) ->
     [];
-check_configs([Config | Configs]) ->
-    [check_config(Config) | check_configs(Configs)].
+check_configs([Config | Configs], Nth) ->
+    [check_config(Config, #{id_for_log => Nth}) | check_configs(Configs, Nth + 1)].
 
 check_config(Config) ->
     check_config(Config, #{}).
@@ -55,15 +58,16 @@ check_config(Config, Opts) ->
         #{?CONF_NS_BINARY := WithDefaults} -> WithDefaults
     end.
 
-do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
+do_check_config(#{<<"mechanism">> := Mec0} = Config, Opts) ->
+    Mec = atom(Mec0, #{error => unknown_mechanism}),
     Key =
         case maps:get(<<"backend">>, Config, false) of
-            false -> atom(Mec);
-            Backend -> {atom(Mec), atom(Backend)}
+            false -> Mec;
+            Backend -> {Mec, atom(Backend, #{error => unknown_backend})}
         end,
     case lists:keyfind(Key, 1, providers()) of
         false ->
-            throw({unknown_handler, Key});
+            throw(#{error => unknown_authn_provider, which => Key});
         {_, ProviderModule} ->
             hocon_tconf:check_plain(
                 ProviderModule,
@@ -71,22 +75,22 @@ do_check_config(#{<<"mechanism">> := Mec} = Config, Opts) ->
                 Opts#{atom_key => true}
             )
     end;
-do_check_config(Config, _Opts) when is_map(Config) ->
-    throw({invalid_config, "mechanism_field_required", Config});
-do_check_config(RawConf, Opts) ->
-    %% authentication conf is lazy type, when it comes from ENV, it is a string
-    %% EMQX_AUTHENTICATION__1="{mechanism=\"password_based\"...}"
-    case hocon:binary(RawConf, Opts) of
-        {ok, Conf} -> do_check_config(Conf, Opts);
-        {error, Reason} -> throw({invalid_config, Reason})
-    end.
+do_check_config(Config, Opts) when is_map(Config) ->
+    throw(#{
+        error => invalid_config,
+        which => maps:get(id_for_log, Opts, unknown),
+        reason => "mechanism_field_required"
+    }).
 
-atom(Bin) ->
+%% The atoms have to be loaded already,
+%% which might be an issue for plugins which are loaded after node boot
+%% but they should really manage their own configs in that case.
+atom(Bin, ErrorContext) ->
     try
         binary_to_existing_atom(Bin, utf8)
     catch
         _:_ ->
-            throw({unknown_auth_provider, Bin})
+            throw(ErrorContext#{value => Bin})
     end.
 
 -spec get_enabled_authns() ->

apps/emqx_bridge/src/emqx_bridge_resource.erl

@@ -298,8 +298,8 @@ parse_confs(Type, Name, Conf) when ?IS_BI_DIR_BRIDGE(Type) ->
     %% For some drivers that can be used as data-sources, we need to provide a
     %% hookpoint. The underlying driver will run `emqx_hooks:run/3` when it
     %% receives a message from the external database.
-    BName = bridge_id(Type, Name),
-    Conf#{hookpoint => <<"$bridges/", BName/binary>>, bridge_name => Name};
+    BId = bridge_id(Type, Name),
+    Conf#{hookpoint => <<"$bridges/", BId/binary>>, bridge_name => Name};
 parse_confs(_Type, _Name, Conf) ->
     Conf.
 

bin/emqx

@@ -115,14 +115,14 @@ usage() {
         echo "Print path to Erlang runtime bin dir"
         ;;
     rpc)
-        echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]"
+        echo "Usage: $REL_NAME rpc MODULE FUNCTION [ARGS, ...]"
         echo "Connect to the EMQX node and make an Erlang RPC"
         echo "This command blocks for at most 60 seconds."
         echo "It exits with non-zero code in case of any RPC failure"
         echo "including connection error and runtime exception"
         ;;
     rpcterms)
-        echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]"
+        echo "Usage: $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]"
         echo "Connect to the EMQX node and make an Erlang RPC"
         echo "The result of the RPC call is pretty-printed as an "
         echo "Erlang term"

changes/v5.0.12-en.md

@@ -36,7 +36,7 @@ Please note, the request body of `/bridges` API to configure MQTT brdige is chan
 
 - Return `204` instead of `200` for `POST /gateway/lwm2m/clients/{clientid}/{read,write,observe}` [#9480](https://github.com/emqx/emqx/pull/9480).
 
-- Make possible to create an authentication entirely from environment variable [#9437](https://github.com/emqx/emqx/pull/9437).
+- Make possible to create an authentication entirely from environment variable [#9547](https://github.com/emqx/emqx/pull/9547).
   As an example, one can now enable MySQL auth with:
   `env EMQX_AUTHENTICATION__1='{mechanism="password_based",backend="mysql",server="localhost:3306",database="emqx",username="emqx",password="******",query="SELECT password_hash,salt FROM mqtt_user WHERE username=${username} LIMIT 1",enable=true}'`.
   Prior to this change, overrides only work on top of existing authentication, for example, if there is already MySQL auth configured in `emqx.conf`

changes/v5.0.12-zh.md

@@ -35,7 +35,7 @@ v5.0.11 或更早版本创建的配置文件，在新版本中会被自动转换
 
 - 现在调用 `POST /gateway/lwm2m/clients/{clientid}/{read,write,observe}` 时，将会返回 204，而不再是 200 [#9480](https://github.com/emqx/emqx/pull/9480)。
 
-- 允许使用环境变量来创建一个认证配置 [#9437](https://github.com/emqx/emqx/pull/9437)。
+- 允许使用环境变量来创建一个认证配置 [#9547](https://github.com/emqx/emqx/pull/9547)。
   例如，现在可以用如下环境变量来创建一个 MySQL 认证：
   `env EMQX_AUTHENTICATION__1='{mechanism="password_based",backend="mysql",server="localhost:3306",database="emqx",username="emqx",password="******",query="SELECT password_hash,salt FROM mqtt_user WHERE username=${username} LIMIT 1",enable=true}'`。
   在此之前，环境变量的重载仅作用于已经存在的配置之上，例如，当 `emqx.conf` 中已经配置了一个 MySQL 认证，那么可以使用如下方法来将它禁用：

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.11
+version: 5.0.12
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.11
+appVersion: 5.0.12

lib-ee/emqx_ee_bridge/rebar.config

@@ -1,5 +1,5 @@
 {erl_opts, [debug_info]}.
-{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}
+{deps, [ {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
        , {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.0"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.0"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}}

mix.exs

@@ -67,7 +67,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.31.2", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.32.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -67,7 +67,7 @@
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.31.2"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.32.0"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}}
     , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}

scripts/get-distro.sh

@@ -14,7 +14,11 @@ case "$UNAME" in
         SYSTEM="${DIST}${VERSION_ID}"
         ;;
     Linux)
-        if grep -q -i 'rhel' /etc/*-release; then
+        # /etc/os-release on amazon linux 2 contains both rhel and centos strings
+        if grep -q -i 'amzn' /etc/*-release; then
+            DIST='amzn'
+            VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+        elif grep -q -i 'rhel' /etc/*-release; then
             DIST='el'
             VERSION_ID="$(rpm --eval '%{rhel}')"
         else