fix: load authz's default sources

apps/emqx/include/emqx.hrl

@@ -17,9 +17,6 @@
 -ifndef(EMQX_HRL).
 -define(EMQX_HRL, true).
 
-%% Config
--define(READ_ONLY_KEYS, [cluster, rpc, node]).
-
 %% Shard
 %%--------------------------------------------------------------------
 -define(COMMON_SHARD, emqx_common_shard).

apps/emqx/src/emqx_config.erl

@@ -35,7 +35,8 @@
     save_to_app_env/1,
     save_to_config_map/2,
     save_to_override_conf/3,
-    reload_etc_conf_on_local_node/0
+    config_files/0,
+    include_dirs/0
 ]).
 -export([merge_envs/2]).
 
@@ -91,6 +92,7 @@
 ]).
 
 -export([ensure_atom_conf_path/2]).
+-export([load_config_files/2]).
 
 -ifdef(TEST).
 -export([erase_all/0, backup_and_write/2]).
@@ -977,94 +979,5 @@ put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
 put_config_post_change_actions(_Key, _NewValue) ->
     ok.
 
-%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
--spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
-reload_etc_conf_on_local_node() ->
-    case load_etc_config_file() of
-        {ok, RawConf} ->
-            case check_readonly_config(RawConf) of
-                {ok, Reloaded} -> reload_config(Reloaded);
-                {error, Error} -> {error, Error}
-            end;
-        {error, _Error} ->
-            {error, bad_hocon_file}
-    end.
-
-reload_config(AllConf) ->
-    Func = fun(Key, Conf, Acc) ->
-        case emqx:update_config([Key], Conf, #{persistent => false}) of
-            {ok, _} ->
-                io:format("Reloaded ~ts config ok~n", [Key]),
-                Acc;
-            Error ->
-                ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
-                ?SLOG(error, #{
-                    msg => "failed_to_reload_etc_config",
-                    key => Key,
-                    value => Conf,
-                    error => Error
-                }),
-                Acc#{Key => Error}
-        end
-    end,
-    Res = maps:fold(Func, #{}, AllConf),
-    case Res =:= #{} of
-        true -> ok;
-        false -> {error, Res}
-    end.
-
-%% @doc Merge etc/emqx.conf on top of cluster.hocon.
-%% For example:
-%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
-%% but `authorization.sources` in not in the default emqx.conf file.
-%% To make sure all root keys in emqx.conf has a fully merged value.
-load_etc_config_file() ->
-    ConfFiles = config_files(),
-    Opts = #{format => map, include_dirs => include_dirs()},
-    case hocon:files(ConfFiles, Opts) of
-        {ok, RawConf} ->
-            HasDeprecatedFile = has_deprecated_file(),
-            %% Merge etc.conf on top of cluster.hocon,
-            %% Don't use map deep_merge, use hocon files merge instead.
-            %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
-            Keys = maps:keys(RawConf),
-            MergedRaw = load_config_files(HasDeprecatedFile, ConfFiles),
-            {ok, maps:with(Keys, MergedRaw)};
-        {error, Error} ->
-            ?SLOG(error, #{
-                msg => "failed_to_read_etc_config",
-                files => ConfFiles,
-                error => Error
-            }),
-            {error, Error}
-    end.
-
-check_readonly_config(Raw) ->
-    SchemaMod = emqx_conf:schema_module(),
-    {_AppEnvs, CheckedConf} = check_config(SchemaMod, fill_defaults(Raw), #{}),
-    case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READ_ONLY_KEYS) of
-        [] ->
-            {ok, maps:without([atom_to_binary(K) || K <- ?READ_ONLY_KEYS], Raw)};
-        Error ->
-            ?SLOG(error, #{
-                msg => "failed_to_change_read_only_key_in_etc_config",
-                read_only_keys => ?READ_ONLY_KEYS,
-                error => Error
-            }),
-            {error, Error}
-    end.
-
-filter_changed(Key, ChangedConf) ->
-    Prev = get([Key], #{}),
-    New = maps:get(Key, ChangedConf, #{}),
-    case Prev =/= New of
-        true -> {true, {Key, changed(New, Prev)}};
-        false -> false
-    end.
-
-changed(New, Prev) ->
-    Diff = emqx_utils_maps:diff_maps(New, Prev),
-    maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
-
 config_files() ->
     application:get_env(emqx, config_files, []).

apps/emqx_authz/src/emqx_authz.erl

@@ -166,8 +166,9 @@ do_pre_config_update(?ROOT_KEY, NewConf, OldConf) ->
 do_pre_config_replace(Conf, Conf) ->
     Conf;
 do_pre_config_replace(NewConf, OldConf) ->
-    NewSources = maps:get(<<"sources">>, NewConf, []),
+    Default = [emqx_authz_schema:default_authz()],
-    OldSources = maps:get(<<"sources">>, OldConf, []),
+    NewSources = maps:get(<<"sources">>, NewConf, Default),
+    OldSources = maps:get(<<"sources">>, OldConf, Default),
     NewSources1 = do_pre_config_update({?CMD_REPLACE, NewSources}, OldSources),
     NewConf#{<<"sources">> => NewSources1}.
 

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -42,7 +42,8 @@
 
 -export([
     headers_no_content_type/1,
-    headers/1
+    headers/1,
+    default_authz/0
 ]).
 
 %%--------------------------------------------------------------------

apps/emqx_conf/include/emqx_conf.hrl

@@ -34,4 +34,6 @@
     tnx_id :: pos_integer() | '$1'
 }).
 
+-define(READONLY_KEYS, [cluster, rpc, node]).
+
 -endif.

apps/emqx_conf/src/emqx_conf.erl

@@ -19,6 +19,7 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/emqx_schema.hrl").
+-include("emqx_conf.hrl").
 
 -export([add_handler/2, remove_handler/1]).
 -export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]).
@@ -30,6 +31,7 @@
 -export([dump_schema/2]).
 -export([schema_module/0]).
 -export([gen_example_conf/2]).
+-export([reload_etc_conf_on_local_node/0]).
 
 %% TODO: move to emqx_dashboard when we stop building api schema at build time
 -export([
@@ -213,10 +215,100 @@ schema_module() ->
         Value -> list_to_existing_atom(Value)
     end.
 
+%% @doc Reload etc/emqx.conf to runtime config except for the readonly config
+-spec reload_etc_conf_on_local_node() -> ok | {error, term()}.
+reload_etc_conf_on_local_node() ->
+    case load_etc_config_file() of
+        {ok, RawConf} ->
+            case check_readonly_config(RawConf) of
+                {ok, Reloaded} -> reload_config(Reloaded);
+                {error, Error} -> {error, Error}
+            end;
+        {error, _Error} ->
+            {error, bad_hocon_file}
+    end.
+
+reload_config(AllConf) ->
+    Func = fun(Key, Conf, Acc) ->
+        case emqx:update_config([Key], Conf, #{persistent => false}) of
+            {ok, _} ->
+                io:format("Reloaded ~ts config ok~n", [Key]),
+                Acc;
+            Error ->
+                ?ELOG("Reloaded ~ts config failed~n~p~n", [Key, Error]),
+                ?SLOG(error, #{
+                    msg => "failed_to_reload_etc_config",
+                    key => Key,
+                    value => Conf,
+                    error => Error
+                }),
+                Acc#{Key => Error}
+        end
+    end,
+    Res = maps:fold(Func, #{}, AllConf),
+    case Res =:= #{} of
+        true -> ok;
+        false -> {error, Res}
+    end.
+
+%% @doc Merge etc/emqx.conf on top of cluster.hocon.
+%% For example:
+%% `authorization.sources` will be merged into cluster.hocon when updated via dashboard,
+%% but `authorization.sources` in not in the default emqx.conf file.
+%% To make sure all root keys in emqx.conf has a fully merged value.
+load_etc_config_file() ->
+    ConfFiles = emqx_config:config_files(),
+    Opts = #{format => map, include_dirs => emqx_config:include_dirs()},
+    case hocon:files(ConfFiles, Opts) of
+        {ok, RawConf} ->
+            HasDeprecatedFile = emqx_config:has_deprecated_file(),
+            %% Merge etc.conf on top of cluster.hocon,
+            %% Don't use map deep_merge, use hocon files merge instead.
+            %% In order to have a chance to delete. (e.g. zones.zone1.mqtt = null)
+            Keys = maps:keys(RawConf),
+            MergedRaw = emqx_config:load_config_files(HasDeprecatedFile, ConfFiles),
+            {ok, maps:with(Keys, MergedRaw)};
+        {error, Error} ->
+            ?SLOG(error, #{
+                msg => "failed_to_read_etc_config",
+                files => ConfFiles,
+                error => Error
+            }),
+            {error, Error}
+    end.
+
+check_readonly_config(Raw) ->
+    SchemaMod = schema_module(),
+    RawDefault = emqx_config:fill_defaults(Raw),
+    {_AppEnvs, CheckedConf} = emqx_config:check_config(SchemaMod, RawDefault),
+    case lists:filtermap(fun(Key) -> filter_changed(Key, CheckedConf) end, ?READONLY_KEYS) of
+        [] ->
+            {ok, maps:without([atom_to_binary(K) || K <- ?READONLY_KEYS], Raw)};
+        Error ->
+            ?SLOG(error, #{
+                msg => "failed_to_change_read_only_key_in_etc_config",
+                read_only_keys => ?READONLY_KEYS,
+                error => Error
+            }),
+            {error, Error}
+    end.
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
 
+filter_changed(Key, ChangedConf) ->
+    Prev = get([Key], #{}),
+    New = maps:get(Key, ChangedConf, #{}),
+    case Prev =/= New of
+        true -> {true, {Key, changed(New, Prev)}};
+        false -> false
+    end.
+
+changed(New, Prev) ->
+    Diff = emqx_utils_maps:diff_maps(New, Prev),
+    maps:filter(fun(_Key, Value) -> Value =/= #{} end, maps:remove(identical, Diff)).
+
 %% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump.
 make_desc_resolver(Lang) ->
     fun

apps/emqx_conf/src/emqx_conf_cli.erl

@@ -15,7 +15,7 @@
 %%--------------------------------------------------------------------
 
 -module(emqx_conf_cli).
--include_lib("emqx/include/emqx.hrl").
+-include("emqx_conf.hrl").
 -export([
     load/0,
     admins/1,
@@ -47,6 +47,8 @@ conf(["load", Path]) ->
     load_config(Path);
 conf(["cluster_sync" | Args]) ->
     admins(Args);
+conf(["reload"]) ->
+    emqx_conf:reload_etc_conf_on_local_node();
 conf(_) ->
     emqx_ctl:usage(usage_conf() ++ usage_sync()).
 
@@ -175,6 +177,9 @@ get_config(Key) ->
 -define(OPTIONS, #{rawconf_with_defaults => true, override_to => cluster}).
 load_config(Path) ->
     case hocon:files([Path]) of
+        {ok, RawConf} when RawConf =:= #{} ->
+            emqx_ctl:warning("load ~ts is empty~n", [Path]),
+            {error, empty_hocon_file};
         {ok, RawConf} ->
             case check_config_keys(RawConf) of
                 ok ->
@@ -200,8 +205,8 @@ update_config(Key, Value) ->
     end.
 check_config_keys(Conf) ->
     Keys = maps:keys(Conf),
-    ReadOnlyKeys = [atom_to_binary(K) || K <- ?READ_ONLY_KEYS],
+    ReadOnlyKeys = [atom_to_binary(K) || K <- ?READONLY_KEYS],
     case ReadOnlyKeys -- Keys of
         ReadOnlyKeys -> ok;
-        _ -> {error, "update_read_only_keys_prohibited"}
+        _ -> {error, "update_readonly_keys_prohibited"}
     end.

apps/emqx_conf/test/emqx_conf_cli_SUITE.erl

@@ -0,0 +1,72 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_conf_cli_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include("emqx_conf.hrl").
+-import(emqx_config_SUITE, [prepare_conf_file/3]).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authz]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authz]).
+
+t_load_config(Config) ->
+    Authz = authorization,
+    Conf = emqx_conf:get_raw([Authz]),
+    %% set sources to []
+    ConfBin0 = hocon_pp:do(#{<<"authorization">> => Conf#{<<"sources">> => []}}, #{}),
+    ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
+    ok = emqx_conf_cli:conf(["load", ConfFile0]),
+    ?assertEqual(Conf#{<<"sources">> := []}, emqx_conf:get_raw([Authz])),
+    %% remove sources, it will reset to default file source.
+    ConfBin1 = hocon_pp:do(#{<<"authorization">> => maps:remove(<<"sources">>, Conf)}, #{}),
+    ConfFile1 = prepare_conf_file(?FUNCTION_NAME, ConfBin1, Config),
+    ok = emqx_conf_cli:conf(["load", ConfFile1]),
+    Default = [emqx_authz_schema:default_authz()],
+    ?assertEqual(Conf#{<<"sources">> := Default}, emqx_conf:get_raw([Authz])),
+    %% reset
+    ConfBin2 = hocon_pp:do(#{<<"authorization">> => Conf}, #{}),
+    ConfFile2 = prepare_conf_file(?FUNCTION_NAME, ConfBin2, Config),
+    ok = emqx_conf_cli:conf(["load", ConfFile2]),
+    ?assertEqual(Conf, emqx_conf:get_raw([Authz])),
+    ?assertEqual({error, empty_hocon_file}, emqx_conf_cli:conf(["load", "non-exist-file"])),
+    ok.
+
+t_load_readonly(Config) ->
+    Base = #{<<"mqtt">> => emqx_conf:get_raw([mqtt])},
+    lists:foreach(
+        fun(Key) ->
+            Conf = emqx_conf:get_raw([Key]),
+            ConfBin0 = hocon_pp:do(Base#{Key => Conf}, #{}),
+            ConfFile0 = prepare_conf_file(?FUNCTION_NAME, ConfBin0, Config),
+            ?assertEqual(
+                {error, "update_readonly_keys_prohibited"},
+                emqx_conf_cli:conf(["load", ConfFile0])
+            )
+        end,
+        ?READONLY_KEYS
+    ),
+    ok.