Merge pull request #4987 from zmstone/pin-hocon-0.5.1

fix(hocon): pin hocon 0.5.1
This commit is contained in:
Zaiming (Stone) Shi 2021-06-15 13:15:52 +02:00 committed by GitHub
commit b1137b8459
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 86 additions and 65 deletions

View File

@ -37,12 +37,16 @@ emqx_test(){
"zip")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
unzip -q "${PACKAGE_PATH}/${packagename}"
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
echo "running ${packagename} start"
"${PACKAGE_PATH}"/emqx/bin/emqx start || ( tail "${PACKAGE_PATH}"/emqx/log/emqx.log.1 && exit 1 )
if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then
cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! curl http://localhost:8081/status >/dev/null 2>&1; do
if [ $IDLE_TIME -gt 10 ]
@ -109,11 +113,15 @@ emqx_test(){
}
running_test(){
export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
if ! emqx start; then
cat /var/log/emqx/erlang.log.1 || true
cat /var/log/emqx/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! curl http://localhost:8081/status >/dev/null 2>&1; do
if [ $IDLE_TIME -gt 10 ]
@ -130,7 +138,12 @@ running_test(){
if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
|| [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
if ! service emqx start; then
cat /var/log/emqx/erlang.log.1 || true
cat /var/log/emqx/emqx.log.1 || true
exit 1
fi
IDLE_TIME=0
while ! curl http://localhost:8081/status >/dev/null 2>&1; do
if [ $IDLE_TIME -gt 10 ]
@ -154,7 +167,11 @@ relup_test(){
while read -r pkg; do
packagename=$(basename "${pkg}")
unzip "$packagename"
./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
if ! ./emqx/bin/emqx start; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases

View File

@ -46,6 +46,7 @@ jobs:
- name: run test cases
run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
export HOCON_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_ldap"
@ -115,6 +116,7 @@ jobs:
- name: run test cases
run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
export HOCON_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mongo"
@ -197,6 +199,7 @@ jobs:
- name: run test cases
run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
export HOCON_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mysql"
@ -270,7 +273,8 @@ jobs:
export EMQX_AUTH__PGSQL__USERNAME=root \
EMQX_AUTH__PGSQL__PASSWORD=public \
EMQX_AUTH__PGSQL__DATABASE=mqtt \
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ \
HOCON_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_pgsql"

View File

@ -37,6 +37,7 @@ jobs:
run: |
set -e -u -x
echo "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
docker-compose \

View File

@ -16,7 +16,7 @@
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.0"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}}
, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}}
, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}

View File

@ -1,6 +1,5 @@
-module(emqx_schema).
% tmp
-dialyzer(no_return).
-dialyzer(no_match).
-dialyzer(no_contracts).
@ -106,14 +105,14 @@ fields("rlog") ->
];
fields("node") ->
[ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "NODE_NAME")}
[ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "EMQX_NODE_NAME")}
, {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)}
, {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "NODE_COOKIE")}
, {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "EMQX_NODE_COOKIE")}
, {"data_dir", t(string(), "emqx.data_dir", undefined)}
, {"heartbeat", t(flag(), undefined, false)}
, {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)}
, {"process_limit", t(integer(), "vm_args.+P", undefined)}
, {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "MAX_PORTS")}
, {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "EMQX_MAX_PORTS")}
, {"dist_buffer_size", fun node__dist_buffer_size/1}
, {"global_gc_interval", t(duration_s(), "emqx.global_gc_interval", undefined)}
, {"fullsweep_after", t(non_neg_integer(),
@ -204,7 +203,7 @@ fields("acl") ->
];
fields("mqtt") ->
[ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "MAX_PACKET_SIZE")}
[ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "EMQX_MAX_PACKET_SIZE")}
, {"max_clientid_len", t(integer(), "emqx.max_clientid_len", 65535)}
, {"max_topic_levels", t(integer(), "emqx.max_topic_levels", 0)}
, {"max_qos_allowed", t(range(0, 2), "emqx.max_qos_allowed", 2)}
@ -1180,18 +1179,16 @@ ceiling(X) ->
%% types
t(T) ->
fun (type) -> T; (_) -> undefined end.
t(Type) -> hoconsc:t(Type).
t(T, M, D) ->
fun (type) -> T; (mapping) -> M; (default) -> D; (_) -> undefined end.
t(Type, Mapping, Default) ->
hoconsc:t(Type, #{mapping => Mapping, default => Default}).
t(T, M, D, O) ->
fun (type) -> T;
(mapping) -> M;
(default) -> D;
(override_env) -> O;
(_) -> undefined end.
t(Type, Mapping, Default, OverrideEnv) ->
hoconsc:t(Type, #{ mapping => Mapping
, default => Default
, override_env => OverrideEnv
}).
ref(Field) ->
fun (type) -> Field; (_) -> undefined end.

View File

@ -18,7 +18,8 @@
-include_lib("typerefl/include/types.hrl").
-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
-export([ schema/0
-export([ structs/0
, fields/1
]).
%% callbacks of behaviour emqx_resource
@ -34,6 +35,10 @@
-export([do_health_check/1]).
%%=====================================================================
structs() -> ["config"].
fields("config") -> schema().
schema() ->
emqx_connector_schema_lib:relational_db_fields() ++
emqx_connector_schema_lib:ssl_fields().

View File

@ -7,7 +7,7 @@
{mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [
{datatype, string},
{override_env, "ADMIN_PASSWORD"}
{override_env, "EMQX_ADMIN_PASSWORD"}
]}.
{mapping, "dashboard.listener.http.port", "emqx_dashboard.listeners", [

View File

@ -18,21 +18,6 @@
-export([check/2]).
-export([structs/0, fields/1]).
-behaviour(hocon_schema).
check(SchemaMod, Conf) ->
_ = erlang:erase(res_schema_mod),
erlang:put(res_schema_mod, SchemaMod),
hocon_schema:check(?MODULE, Conf).
hocon_schema:check(SchemaMod, Conf, #{nullable => false}).
structs() -> ["config"].
fields("config") ->
[fun(type) -> "schema";
(_) -> undefined
end];
fields("schema") ->
SchemaMod = erlang:get(res_schema_mod),
SchemaMod:schema().

View File

@ -20,6 +20,10 @@ mkdir -p "$RUNNER_LOG_DIR"
# Make sure data directory exists
mkdir -p "$RUNNER_DATA_DIR"
# Make sure data/configs exists
CONFIGS_DIR="$RUNNER_DATA_DIR/configs"
mkdir -p "$CONFIGS_DIR"
# hocon try to read environment variables starting with "EMQX_"
export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
@ -204,39 +208,48 @@ generate_config() {
EMQX_LICENSE_CONF_OPTION="-c ${EMQX_LICENSE_CONF}"
fi
set +e
# disable shellcheck; let EMQX_LICENSE_CONF_OPTION split
## timestamp for each generation
local NOW_TIME
NOW_TIME="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon now_time)"
## ths command populates two files: app.<time>.config and vm.<time>.args
## disable SC2086 to allow EMQX_LICENSE_CONF_OPTION to split
# shellcheck disable=SC2086
HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf $EMQX_LICENSE_CONF_OPTION -d "$RUNNER_DATA_DIR"/configs generate)"
# shellcheck disable=SC2181
RESULT=$?
set -e
if [ $RESULT -gt 0 ]; then
echo "$HOCON_OUTPUT"
exit $RESULT
fi
# print override from environment variables (EMQX_*)
echo "$HOCON_OUTPUT" | sed -e '$d'
CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1)
"$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -t "$NOW_TIME" -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf $EMQX_LICENSE_CONF_OPTION -d "$RUNNER_DATA_DIR"/configs generate
## filenames are per-hocon convention
local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
local HOCON_GEN_ARG_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args"
CONFIG_ARGS="-config $CONF_FILE -args_file $HOCON_GEN_ARG_FILE"
## Merge hocon generated *.args into the vm.args
HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
TMP_ARG_FILE="$CONFIGS_DIR/vm.args.tmp"
cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
echo "" >> "$TMP_ARG_FILE"
echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
## read lines from generated vm.<time>.args file
## drop comment lines, and empty lines using sed
## pipe the lines to a while loop
sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
## in the loop, split the 'key[:space:]value' pair
ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
## use the key to look up in vm.args file for the value
TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
## compare generated (to override) value to original (to be overriden) value
if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
## if they are different
if [ -n "$TMP_ARG_VALUE" ]; then
## if the old value is present, replace it with generated value
sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE"
else
## otherwise append generated value to the end
echo "$ARG_LINE" >> "$TMP_ARG_FILE"
fi
fi
done
## rename the generated vm.<time>.args file
mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE"
# shellcheck disable=SC2086
@ -284,9 +297,9 @@ if [ -z "$NAME_ARG" ]; then
if [ "$IS_BOOT_COMMAND" = 'no' ]; then
# for non-boot commands, inspect vm.<time>.args for node name
# shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echo "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
echo "For command $1, there is no vm.*.args config file found in $CONFIGS_DIR/"
exit 1
fi
NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
@ -322,9 +335,9 @@ if [ -z "$COOKIE" ]; then
COOKIE="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")"
else
# shellcheck disable=SC2012,SC2086
LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"
LATEST_VM_ARGS="$(ls -t $CONFIGS_DIR/vm.*.args | head -1)"
if [ -z "$LATEST_VM_ARGS" ]; then
echo "For command $1, there is no vm.*.args config file found in $RUNNER_DATA_DIR/configs/"
echo "For command $1, there is no vm.*.args config file found in $CONFIGS_DIR/"
exit 1
fi
COOKIE="$(grep -E '^-setcookie' "$LATEST_VM_ARGS" | awk '{print $2}')"
@ -350,7 +363,7 @@ export BINDIR="$ERTS_DIR/bin"
export EMU="beam"
export PROGNAME="erl"
export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
ERTS_LIB_DIR="$ERTS_DIR/../lib"
export ERTS_LIB_DIR="$ERTS_DIR/../lib"
MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME"
cd "$ROOTDIR"

View File

@ -37,7 +37,6 @@ relx_nodetool() {
command="$1"; shift
ERL_FLAGS="$ERL_FLAGS $EPMD_ARG $PROTO_DIST_ARG" \
ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \
"$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
-setcookie "$COOKIE" "$command" "$@"
}

View File

@ -43,7 +43,7 @@ Use the environment variable to configure the EMQ X docker container.
By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files.
You can change the prefix by overriding "CUTTLEFISH_ENV_OVERRIDE_PREFIX".
You can change the prefix by overriding "HOCON_ENV_OVERRIDE_PREFIX".
Example:
@ -56,7 +56,7 @@ EMQX_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size
+ All upper case letters is replaced with lower case letters
+ ``__`` is replaced with ``.``
If `CUTTLEFISH_ENV_OVERRIDE_PREFIX=DEV_` is set:
If `HOCON_ENV_OVERRIDE_PREFIX=DEV_` is set:
```bash
DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors

View File

@ -52,7 +52,7 @@
, {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
, {getopt, "1.0.1"}
, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.0"}}}
, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}}
, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}}
]}.