From 74b7ea34d9ac39548c03b0432fff5310719841c9 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 16 Feb 2021 11:16:03 +0900 Subject: [PATCH] feat(bin): let cuttlefish read env var --- bin/emqx | 3 + deploy/docker/README.md | 29 ++++---- deploy/docker/docker-entrypoint.sh | 73 ------------------- .../emqx_dashboard/priv/emqx_dashboard.schema | 3 +- priv/emqx.schema | 3 +- rebar.config | 2 +- 6 files changed, 22 insertions(+), 91 deletions(-) diff --git a/bin/emqx b/bin/emqx index 025f08ce5..2fe24af23 100755 --- a/bin/emqx +++ b/bin/emqx @@ -20,6 +20,9 @@ mkdir -p "$RUNNER_LOG_DIR" # Make sure data directory exists mkdir -p "$RUNNER_DATA_DIR" +# cuttlefish try to read environment variables starting with "EMQX_", if not specified +export CUTTLEFISH_ENV_OVERRIDE_PREFIX="${CUTTLEFISH_ENV_OVERRIDE_PREFIX:-EMQX_}" + relx_usage() { command="$1" diff --git a/deploy/docker/README.md b/deploy/docker/README.md index d0b88331e..5ea42aadc 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -41,11 +41,9 @@ The emqx broker runs as linux user `emqx` in the docker container. Use the environment variable to configure the EMQ X docker container. -The environment variables which with ``EMQX_`` prefix are mapped to configuration fils. +By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files. -+ Prefix ``EMQX_`` is removed -+ All upper case letters is replaced with lower case letters -+ ``__`` is replaced with ``.`` +You can change the prefix by overriding "CUTTLEFISH_ENV_OVERRIDE_PREFIX". Example: @@ -54,6 +52,17 @@ EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors EMQX_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size ``` ++ Prefix ``EMQX_`` is removed ++ All upper case letters is replaced with lower case letters ++ ``__`` is replaced with ``.`` + +If `CUTTLEFISH_ENV_OVERRIDE_PREFIX=DEV_` is set: + +```bash +DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors +DEV_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size +``` + Non mapped environment variables: ```bash @@ -189,16 +198,6 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \ emqx/emqx:latest ``` -#### Mask Sensitive Configuration - -Use ``MASK_CONFIG_FILTER`` to hide senstive configuration values from leaking to logging system. - -For example, set ``MASK_CONFIG_FILTER="password|token"`` to hide all configuration names containing those keywords. - -By default emqx masks the configuration using following filter `"password|passwd|key|token|secret"`. Setting ``MASK_CONFIG_FILTER`` will be merged with the default filter. - -The configuration should match whole word (after splitting it by '.') with `MASK_CONFIG_FILTER`. You can use commas, spaces or other required separators to separate different words. - ### Cluster EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details. @@ -234,7 +233,7 @@ Let's create a static node list cluster from docker-compose. emqx-bridge: aliases: - node2.emqx.io - + networks: emqx-bridge: driver: bridge diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 2934f3a78..91e474818 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -90,79 +90,6 @@ if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS" ]]; then export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400 fi -# Fix issue #42 - export env EMQX_DASHBOARD__DEFAULT_USER__PASSWORD to configure -# 'dashboard.default_user.password' in etc/plugins/emqx_dashboard.conf -if [[ -n "$EMQX_ADMIN_PASSWORD" ]]; then - export EMQX_DASHBOARD__DEFAULT_USER__PASSWORD=$EMQX_ADMIN_PASSWORD -fi - -# echo value of $VAR hiding secrets if any -# SYNOPSIS -# echo_value KEY VALUE -echo_value() { - # get MASK_CONFIG - MASK_CONFIG_FILTER="$MASK_CONFIG_FILTER|password|passwd|key|token|secret" - FORMAT_MASK_CONFIG_FILTER=$(echo "$MASK_CONFIG_FILTER" | sed -r -e 's/^[^A-Za-z0-9_]+//' -e 's/[^A-Za-z0-9_]+$//' -e 's/[^A-Za-z0-9_]+/|/g') - local key=$1 - local value=$2 - # check if contains sensitive value - if echo "$key" | grep -iqwE "$FORMAT_MASK_CONFIG_FILTER"; then - echo "$key=***secret***" - else - echo "$key=$value" - fi -} - -# fill config on specific file if the key exists -# SYNOPSIS -# try_fill_config FILE KEY VALUE -try_fill_config() { - local file=$1 - local key=$2 - local value=$3 - local escaped_key - # shellcheck disable=SC2001 - escaped_key=$(echo "$key" | sed 's/[^a-zA-Z0-9_]/\\&/g') - local escaped_value - escaped_value=$(echo "$value" | sed 's/[\/&]/\\&/g') - if grep -qE "^[#[:space:]]*$escaped_key\s*=" "$file"; then - echo_value "$key" "$value" - if [[ -z "$value" ]]; then - sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file" > tmpfile && cat tmpfile > "$file" - else - sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file" > tmpfile && cat tmpfile > "$file" - fi - # Check if config has a numbering system, but no existing configuration line in file - elif echo "$key" | grep -qE '\.\d+|\d+\.'; then - if [[ -n "$value" ]]; then - local template - template="$(echo "$escaped_key" | sed -r -e 's/\\\.[0-9]+/\\.[0-9]+/g' -e 's/[0-9]+\\\./[0-9]+\\./g')" - if grep -qE "^[#[:space:]]*$template\s*=" "$file"; then - echo_value "$key" "$value" - sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file" - echo "$key = $value" >> "$file" - fi - fi - fi -} - -# Catch all EMQX_ prefix environment variable and match it in configure file -CONFIG_FILE="$_EMQX_HOME/etc/emqx.conf" -CONFIG_PLUGINS="$_EMQX_HOME/etc/plugins" -for VAR in $(compgen -e); do - # Config normal keys such like node.name = emqx@127.0.0.1 - if echo "$VAR" | grep -q '^EMQX_'; then - VAR_NAME=$(echo "$VAR" | sed -e 's/^EMQX_//' -e 's/__/./g' | tr '[:upper:]' '[:lower:]' | tr -d '[:cntrl:]') - VAR_VALUE=$(echo "${!VAR}" | tr -d '[:cntrl:]') - # Config in emqx.conf - try_fill_config "$CONFIG_FILE" "$VAR_NAME" "$VAR_VALUE" - # Config in plugins/* - for CONFIG_PLUGINS_FILE in "$CONFIG_PLUGINS"/*; do - try_fill_config "$CONFIG_PLUGINS_FILE" "$VAR_NAME" "$VAR_VALUE" - done - fi -done - # fill tuples on specific file # SYNOPSIS # fill_tuples FILE [ELEMENTS ...] diff --git a/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema b/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema index fcc8f3489..d517a6e97 100644 --- a/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema @@ -6,7 +6,8 @@ ]}. {mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [ - {datatype, string} + {datatype, string}, + {override_env, "ADMIN_PASSWORD"} ]}. {mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ diff --git a/priv/emqx.schema b/priv/emqx.schema index bdf8a053f..0ddc02dc9 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -202,7 +202,8 @@ end}. %% @doc Node name {mapping, "node.name", "vm_args.-name", [ - {default, "emqx@127.0.0.1"} + {default, "emqx@127.0.0.1"}, + {override_env, "NODE_NAME"} ]}. %% @doc Specify SSL Options in the file if using SSL for erlang distribution diff --git a/rebar.config b/rebar.config index 76dcd8e98..e35ac8426 100644 --- a/rebar.config +++ b/rebar.config @@ -46,7 +46,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.1.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.1"}}}