yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(conf): use hocon schema

This commit is contained in:
z8674558 2021-05-25 17:25:25 +09:00
parent d61e931d88
commit 4b174b0277
7 changed files with 1477 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.ci/docker-compose-file/conf.env
View File

@ -11,3 +11,4 @@ EMQX_AUTH__PGSQL__DATABASE=mqtt
EMQX_AUTH__REDIS__SERVER=redis_server:6379 EMQX_AUTH__REDIS__SERVER=redis_server:6379
EMQX_AUTH__REDIS__PASSWORD=public EMQX_AUTH__REDIS__PASSWORD=public
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
HOCON_ENV_OVERRIDE_PREFIX=EMQX_

31
bin/emqx
View File

@ -20,8 +20,8 @@ mkdir -p "$RUNNER_LOG_DIR"
# Make sure data directory exists # Make sure data directory exists
mkdir -p "$RUNNER_DATA_DIR" mkdir -p "$RUNNER_DATA_DIR"
# cuttlefish try to read environment variables starting with "EMQX_" # hocon try to read environment variables starting with "EMQX_"
export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
relx_usage() { relx_usage() {
command="$1" command="$1"
@ -123,8 +123,8 @@ fi
# Echo to stderr on errors # Echo to stderr on errors
echoerr() { echo "$@" 1>&2; } echoerr() { echo "$@" 1>&2; }
# By default, use cuttlefish to generate app.config and vm.args # By default, use hocon to generate app.config and vm.args
CUTTLEFISH="${USE_CUTTLEFISH:-yes}" HOCON="${USE_HOCON:-yes}"
SED_REPLACE="sed -i " SED_REPLACE="sed -i "
case $(sed --help 2>&1) in case $(sed --help 2>&1) in
@ -202,7 +202,7 @@ generate_config() {
## changing the config 'log.rotation.size' ## changing the config 'log.rotation.size'
rm -rf "${RUNNER_LOG_DIR}"/*.siz rm -rf "${RUNNER_LOG_DIR}"/*.siz
if [ "$CUTTLEFISH" != "yes" ]; then if [ "$HOCON" != "yes" ]; then
# Note: we have added a parameter '-vm_args' to this. It # Note: we have added a parameter '-vm_args' to this. It
# appears redundant but it is not! the erlang vm allows us to # appears redundant but it is not! the erlang vm allows us to
# access all arguments to the erl command EXCEPT '-args_file', # access all arguments to the erl command EXCEPT '-args_file',
@ -217,25 +217,26 @@ generate_config() {
set +e set +e
# shellcheck disable=SC2086 # shellcheck disable=SC2086
CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
echo $HOCON_OUTPUT
# shellcheck disable=SC2181 # shellcheck disable=SC2181
RESULT=$? RESULT=$?
set -e set -e
if [ $RESULT -gt 0 ]; then if [ $RESULT -gt 0 ]; then
echo "$CUTTLEFISH_OUTPUT" echo "$HOCON_OUTPUT"
exit $RESULT exit $RESULT
fi fi
# print override from environment variables (EMQX_*) # print override from environment variables (EMQX_*)
echo "$CUTTLEFISH_OUTPUT" | sed -e '$d' echo "$HOCON_OUTPUT" | sed -e '$d'
CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1) CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1)
## Merge cuttlefish generated *.args into the vm.args ## Merge hocon generated *.args into the vm.args
CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
echo "" >> "$TMP_ARG_FILE" echo "" >> "$TMP_ARG_FILE"
echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
@ -247,7 +248,7 @@ generate_config() {
fi fi
fi fi
done done
mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE"
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
@ -303,7 +304,7 @@ if [ -z "$NAME_ARG" ]; then
NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')" NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
else else
# for boot commands, inspect emqx.conf for node name # for boot commands, inspect emqx.conf for node name
NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name) NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \")
fi fi
fi fi
if [ -z "$NODENAME" ]; then if [ -z "$NODENAME" ]; then
@ -329,7 +330,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
COOKIE="${EMQX_NODE_COOKIE:-}" COOKIE="${EMQX_NODE_COOKIE:-}"
if [ -z "$COOKIE" ]; then if [ -z "$COOKIE" ]; then
if [ "$IS_BOOT_COMMAND" = 'yes' ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie) COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
else else
# shellcheck disable=SC2012,SC2086 # shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"

22
etc/emqx.conf
View File

@ -598,42 +598,42 @@ log.rotation.count = 5
## Notice: Disable the option in production deployment! ## Notice: Disable the option in production deployment!
## ##
## Value: true | false ## Value: true | false
allow_anonymous = true acl.allow_anonymous = true
## Allow or deny if no ACL rules matched. ## Allow or deny if no ACL rules matched.
## ##
## Value: allow | deny ## Value: allow | deny
acl_nomatch = allow acl.acl_nomatch = allow
## Default ACL File. ## Default ACL File.
## ##
## Value: File Name ## Value: File Name
acl_file = "{{ platform_etc_dir }}/acl.conf" acl.acl_file = "{{ platform_etc_dir }}/acl.conf"
## Whether to enable ACL cache. ## Whether to enable ACL cache.
## ##
## If enabled, ACLs roles for each client will be cached in the memory ## If enabled, ACLs roles for each client will be cached in the memory
## ##
## Value: on | off ## Value: on | off
enable_acl_cache = on acl.enable_acl_cache = on
## The maximum count of ACL entries can be cached for a client. ## The maximum count of ACL entries can be cached for a client.
## ##
## Value: Integer greater than 0 ## Value: Integer greater than 0
## Default: 32 ## Default: 32
acl_cache_max_size = 32 acl.acl_cache_max_size = 32
## The time after which an ACL cache entry will be deleted ## The time after which an ACL cache entry will be deleted
## ##
## Value: Duration ## Value: Duration
## Default: 1 minute ## Default: 1 minute
acl_cache_ttl = 1m acl.acl_cache_ttl = 1m
## The action when acl check reject current operation ## The action when acl check reject current operation
## ##
## Value: ignore | disconnect ## Value: ignore | disconnect
## Default: ignore ## Default: ignore
acl_deny_action = ignore acl.acl_deny_action = ignore
## Specify the global flapping detect policy. ## Specify the global flapping detect policy.
## The value is a string composed of flapping threshold, duration and banned interval. ## The value is a string composed of flapping threshold, duration and banned interval.
@ -642,7 +642,7 @@ acl_deny_action = ignore
## 3. banned interval: the banned interval if a flapping is detected. ## 3. banned interval: the banned interval if a flapping is detected.
## ##
## Value: Integer,Duration,Duration ## Value: Integer,Duration,Duration
flapping_detect_policy = "30, 1m, 5m" acl.flapping_detect_policy = "30, 1m, 5m"
##-------------------------------------------------------------------- ##--------------------------------------------------------------------
## MQTT Protocol ## MQTT Protocol
@ -2155,7 +2155,7 @@ listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1
## The file to store loaded module names. ## The file to store loaded module names.
## ##
## Value: File ## Value: File
modules.loaded_file = "{{ platform_data_dir }}/loaded_modules" module.loaded_file = "{{ platform_data_dir }}/loaded_modules"
##-------------------------------------------------------------------- ##--------------------------------------------------------------------
## Presence Module ## Presence Module
@ -2204,8 +2204,8 @@ module.presence.qos = 1
## Rewrite Module ## Rewrite Module
## {rewrite, Topic, Re, Dest} ## {rewrite, Topic, Re, Dest}
## module.rewrite.pub.rule.1 = "x/# ^x/y/(.+)$ z/y/$1" ## module.rewrite.pub_rule.1 = "x/# ^x/y/(.+)$ z/y/$1"
## module.rewrite.sub.rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" ## module.rewrite.sub_rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2"
## CONFIG_SECTION_END=modules ================================================== ## CONFIG_SECTION_END=modules ==================================================

3
rebar.config
View File

@ -7,7 +7,7 @@
%% rebar.config.rendered if environment DEBUG is set. %% rebar.config.rendered if environment DEBUG is set.
{edoc_opts, [{preprocess,true}]}. {edoc_opts, [{preprocess,true}]}.
{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, {erl_opts, [warn_unused_vars,warn_shadow_vars,
warn_obsolete_guard,compressed, warn_obsolete_guard,compressed,
{d, snk_kind, msg}]}. {d, snk_kind, msg}]}.
@ -56,6 +56,7 @@
, {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
, {getopt, "1.0.1"} , {getopt, "1.0.1"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.2.0"}}}
]}. ]}.
{xref_ignores, {xref_ignores,

5
rebar.config.erl
View File

@ -336,9 +336,8 @@ relx_overlay(ReleaseType) ->
, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
, {copy, "bin/nodetool", "bin/nodetool"} , {copy, "bin/nodetool", "bin/nodetool"}
, {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"} , {copy, "_build/default/lib/hocon/hocon", "bin/hocon"}
, {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"} , {copy, "_build/default/lib/hocon/hocon", "bin/hocon-{{release_version}}"}
, {copy, "priv/emqx.schema", "releases/{{release_version}}/"}
] ++ case is_enterprise() of ] ++ case is_enterprise() of
true -> ee_etc_overlay(ReleaseType); true -> ee_etc_overlay(ReleaseType);
false -> etc_overlay(ReleaseType) false -> etc_overlay(ReleaseType)

1443
src/emqx_schema.erl Normal file
View File

File diff suppressed because it is too large Load Diff

5
test/emqx_listeners_SUITE.erl
View File

@ -65,10 +65,9 @@ mustache_vars() ->
]. ].
generate_config() -> generate_config() ->
Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]),
ConfFile = render_config_file(), ConfFile = render_config_file(),
{ok, Conf} = hocon:load(ConfFile, #{format => proplists}), {ok, Conf} = hocon:load(ConfFile, #{format => richmap}),
cuttlefish_generator:map(Schema, Conf). hocon_schema:generate(emqx_schema, Conf).
set_app_env({App, Lists}) -> set_app_env({App, Lists}) ->
lists:foreach(fun({acl_file, _Var}) -> lists:foreach(fun({acl_file, _Var}) ->