Merge pull request #7148 from zmstone/fix-data-dir-from-config-file

fix(data_dir): get data_dir config in the right way

apps/emqx_conf/etc/emqx_conf.conf

@@ -1,8 +1,8 @@
 ## NOTE:
 ## Configs in this file might be overridden by:
 ## 1. Environment variables which start with 'EMQX_' prefix
-## 2. File {{ platform_data_dir }}/configs/cluster-override.conf
+## 2. File $EMQX_NODE__DATA_DIR/configs/cluster-override.conf
-## 3. File {{ platform_data_dir }}/configs/local-override.conf
+## 3. File $EMQX_NODE__DATA_DIR/configs/local-override.conf
 ##
 ## The *-override.conf files are overwritten at runtime when changes
 ## are made from EMQX dashboard UI, management HTTP API, or CLI.
@@ -30,8 +30,8 @@ node {
   ##
   ## @doc node.data_dir
   ## ValueType: Folder
-  ## Default: "{{ platform_data_dir }}/"
+  ## Default: "{{ platform_data_dir }}"
-  data_dir = "{{ platform_data_dir }}/"
+  data_dir = "{{ platform_data_dir }}"
 
   ## Location of crash dump file.
   ##

apps/emqx_conf/etc/emqx_conf.md

@@ -9,14 +9,18 @@ From bottom up:
 
 1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.<br>
    Changes in this layer require a full node restart to take effect.
-1. Cluster overrides: `${data_dir}/cluster-override.conf`
+1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
-1. Local node overrides: `${data_dir}/configs/local-override.conf`
+1. Local node overrides: `$EMQX_NODE__DATA_DIR/configs/local-override.conf`
 
-Where `${data_dir}` is configurable from `node.data_dir`.
+When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir`
+is used.
 
 The `*-override.conf` files are overwritten at runtime when changes
 are made from dashboard UI, management HTTP API, or CLI.
 
+**NOTE** Config values from `*-override.conf` are **not** mapped to boot configs for
+the config feilds attributed with `mapping: path.to.boot.config.key`
+
 For detailed override rules, see [Config Overlay Rules](#config-overlay-rules).
 
 ## Syntax

apps/emqx_conf/src/emqx_conf.erl

@@ -25,7 +25,7 @@
 -export([update/3, update/4]).
 -export([remove/2, remove/3]).
 -export([reset/2, reset/3]).
--export([dump_schema/1]).
+-export([dump_schema/1, dump_schema/2]).
 
 %% for rpc
 -export([get_node_and_config/1]).
@@ -125,14 +125,17 @@ reset(Node, KeyPath, Opts) ->
 %% @doc Called from build script.
 -spec dump_schema(file:name_all()) -> ok.
 dump_schema(Dir) ->
+    dump_schema(Dir, emqx_conf_schema).
+
+dump_schema(Dir, SchemaModule) ->
     SchemaMdFile = filename:join([Dir, "config.md"]),
     io:format(user, "===< Generating: ~s~n", [SchemaMdFile ]),
-    ok = gen_doc(SchemaMdFile),
+    ok = gen_doc(SchemaMdFile, SchemaModule),
 
     %% for scripts/spellcheck.
     SchemaJsonFile = filename:join([Dir, "schema.json"]),
     io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]),
-    JsonMap = hocon_schema_json:gen(emqx_conf_schema),
+    JsonMap = hocon_schema_json:gen(SchemaModule),
     IoData = jsx:encode(JsonMap, [space, {indent, 4}]),
     ok = file:write_file(SchemaJsonFile, IoData),
 
@@ -146,13 +149,13 @@ dump_schema(Dir) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
--spec gen_doc(file:name_all()) -> ok.
+-spec gen_doc(file:name_all(), module()) -> ok.
-gen_doc(File) ->
+gen_doc(File, SchemaModule) ->
     Version = emqx_release:version(),
     Title = "# EMQX " ++ Version ++ " Configuration",
     BodyFile = filename:join([code:lib_dir(emqx_conf), "etc", "emqx_conf.md"]),
     {ok, Body} = file:read_file(BodyFile),
-    Doc = hocon_schema_md:gen(emqx_conf_schema, #{title => Title, body => Body}),
+    Doc = hocon_schema_md:gen(SchemaModule, #{title => Title, body => Body}),
     file:write_file(File, Doc).
 
 check_cluster_rpc_result(Result) ->

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -264,7 +264,21 @@ fields("node") ->
        sc(string(),
           #{ required => true,
              mapping => "emqx.data_dir",
-             desc => "Path to the persistent data directory. It must be unique per broker instance."
+             desc =>
+"""
+Path to the persistent data directory.
+Possible auto-created sub-directories are:
+  - `mnesia/\<node_name>`: EMQX's built-in database directory.
+    For example, `mnesia/emqx@127.0.0.1`.
+    There should be only one one such sub dirrectory.
+    Meaning, in case the node is to be renamed (to e.g. `emqx@10.0.1.1`),
+    the old dir should be deleted first.
+  - `configs`: Generated configs at boot time, and cluster/local override configs.
+  - `patches`: Hot-patch beam files are to be placed here.
+  - `trace`: Trace log files.
+
+**NOTE**: One data dir can not be shared by two or more EMQX nodes.
+"""
            })}
     , {"config_files",
        sc(list(string()),

apps/emqx_conf/test/emqx_conf_schema_tests.erl

@@ -0,0 +1,13 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_conf_schema_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+doc_gen_test() ->
+    Dir = "tmp",
+    ok = filelib:ensure_dir(filename:join("tmp", foo)),
+    _ = emqx_conf:dump_schema(Dir),
+    ok.

bin/emqx

@@ -28,13 +28,6 @@ WHOAMI=$(whoami)
 # Make sure log directory exists
 mkdir -p "$RUNNER_LOG_DIR"
 
-# Make sure data directory exists
-mkdir -p "$RUNNER_DATA_DIR"
-
-# Make sure data/configs exists
-CONFIGS_DIR="$RUNNER_DATA_DIR/configs"
-mkdir -p "$CONFIGS_DIR"
-
 # hocon try to read environment variables starting with "EMQX_"
 export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
 
@@ -360,7 +353,7 @@ call_hocon() {
 
 get_config_value() {
     path_to_value="$1"
-    call_hocon -s "$SCHEMA_MOD" -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get "$path_to_value" | tr -d \"
+    call_hocon -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf get "$path_to_value" | tr -d \"
 }
 
 check_license() {
@@ -398,6 +391,15 @@ relx_start_command() {
            "$START_OPTION"
 }
 
+DATA_DIR="$(get_config_value 'node.data_dir')"
+DATA_DIR="${DATA_DIR%/}"
+if [[ $DATA_DIR != /* ]]; then
+    # relative
+    DATA_DIR="${RUNNER_ROOT_DIR}/${DATA_DIR}"
+fi
+CONFIGS_DIR="$DATA_DIR/configs"
+mkdir -p "$CONFIGS_DIR"
+
 # Function to generate app.config and vm.args
 generate_config() {
     local name_type="$1"
@@ -414,7 +416,7 @@ generate_config() {
     ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
     ## but does not include the cluster-override.conf and local-override.conf
     ## meaning, certain overrides will not be mapped to app.<time>.config file
-    call_hocon -v -t "$NOW_TIME" -I "$CONFIGS_DIR/" -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate
+    call_hocon -v -t "$NOW_TIME" -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$DATA_DIR"/configs generate
 
     ## filenames are per-hocon convention
     local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
@@ -573,7 +575,7 @@ fi
 
 # force to use 'emqx' short name
 [ -z "$NAME" ] && NAME='emqx'
-MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME"
+MNESIA_DATA_DIR="$DATA_DIR/mnesia/$NAME"
 
 case "$NAME" in
     *@*)
@@ -585,7 +587,7 @@ esac
 SHORT_NAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
 export ESCRIPT_NAME="$SHORT_NAME"
 
-PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
+PIPE_DIR="${PIPE_DIR:-/$DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
 
 ## make EMQX_NODE_COOKIE right
 if [ -n "${EMQX_NODE_COOKIE:-}" ]; then

lib-ee/emqx_enterprise_conf/test/emqx_enterprise_conf_schema_tests.erl

@@ -0,0 +1,13 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_enterprise_conf_schema_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+doc_gen_test() ->
+    Dir = "tmp",
+    ok = filelib:ensure_dir(filename:join("tmp", foo)),
+    _ = emqx_conf:dump_schema(Dir, emqx_enterprise_conf_schema),
+    ok.

rel/emqx_vars

@@ -11,7 +11,6 @@ RUNNER_BIN_DIR="{{ runner_bin_dir }}"
 RUNNER_LOG_DIR="{{ runner_log_dir }}"
 RUNNER_LIB_DIR="{{ runner_lib_dir }}"
 RUNNER_ETC_DIR="{{ runner_etc_dir }}"
-RUNNER_DATA_DIR="{{ runner_data_dir }}"
 RUNNER_USER="{{ runner_user }}"
 IS_ELIXIR="{{ is_elixir }}"
 SCHEMA_MOD="{{ emqx_schema_mod }}"
@@ -22,6 +21,5 @@ export EMQX_DESCRIPTION='{{ emqx_description }}'
 ## computed vars
 REL_NAME="emqx"
 ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
-RUNNER_DATA_DIR="${EMQX_NODE__DATA_DIR:-$RUNNER_DATA_DIR}"
 
 ## updated vars here