Merge pull request #4205 from z8674558/dev/v4.3.0

merge dev/v4.3.0 to dev/v5.0
2021-02-19 07:28:31 +01:00 · 2021-02-19 07:28:31 +01:00 · 692d08b630
parent 54d82584bc 469c7ad43a
commit 692d08b630
13 changed files with 177 additions and 183 deletions
--- a/.ci/build_packages/tests.sh
+++ b/.ci/build_packages/tests.sh
@ -23,8 +23,8 @@ emqx_test(){
            "zip")
                packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
                unzip -q "${PACKAGE_PATH}/${packagename}"
-                sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
-                sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
+                export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
+                       EMQX_MQTT__MAX_TOPIC_ALIAS=10
                sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins

                echo "running ${packagename} start"
@ -96,8 +96,8 @@ emqx_test(){
 }

 running_test(){
-    sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" /etc/emqx/emqx.conf
-    sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" /etc/emqx/emqx.conf
+    export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
+           EMQX_MQTT__MAX_TOPIC_ALIAS=10
    sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins

    emqx start || tail /var/log/emqx/erlang.log.1
--- a/.ci/fvt_tests/docker-compose.yaml
+++ b/.ci/fvt_tests/docker-compose.yaml
@ -8,7 +8,7 @@ services:
    - "EMQX_NAME=emqx"
    - "EMQX_HOST=node1.emqx.io"
    - "EMQX_CLUSTER__DISCOVERY=static"
-    - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\""
+    - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
    - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
    - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
    command:
@ -35,7 +35,7 @@ services:
    - "EMQX_NAME=emqx"
    - "EMQX_HOST=node2.emqx.io"
    - "EMQX_CLUSTER__DISCOVERY=static"
-    - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\""
+    - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
    - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
    - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
    command:
--- a/.github/workflows/run_cts_tests.yaml
+++ b/.github/workflows/run_cts_tests.yaml
@ -36,18 +36,18 @@ jobs:
      - name: setup
        if: matrix.network_type == 'ipv4'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap)
-          sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address\"|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
+          echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv6'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap)
-          sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address\"|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
+          echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) >> "$GITHUB_ENV"
      - name: run test cases
        run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
          docker exec -i erlang sh -c "make ensure-rebar3"
          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_ldap"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
      - uses: actions/upload-artifact@v1
        if: failure()
        with:
@ -78,30 +78,34 @@ jobs:
        if: matrix.connect_type == 'tls'
        run: |
          docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.mongo.ssl.enable[[:space:]]*=.*|auth.mongo.ssl.enable = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__MONGO__SSL__ENABLE=on
+          EMQX_AUTH__MONGO__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem
+          EMQX_AUTH__MONGO__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem
+          EMQX_AUTH__MONGO__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem
+          EOF
      - name: setup
        env:
          MONGO_TAG: ${{ matrix.mongo_tag }}
        if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d
+          echo EMQX_AUTH__MONGO__SSL__ENABLE=off >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv4'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-          sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = \"$server_address:27017\"|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv6'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo)
-          sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = \"$server_address:27017\"|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
      - name: run test cases
        run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
          docker exec -i erlang sh -c "make ensure-rebar3"
          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_mongo"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
      - uses: actions/upload-artifact@v1
        if: failure()
        with:
@ -132,35 +136,37 @@ jobs:
        if: matrix.connect_type == 'tls'
        run: |
          docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.enable[[:space:]]*=.*|auth.mysql.ssl.enable = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.cacertfile[[:space:]]*=.*|auth.mysql.ssl.cacertfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.certfile[[:space:]]*=.*|auth.mysql.ssl.certfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.keyfile[[:space:]]*=.*|auth.mysql.ssl.keyfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+            EMQX_AUTH__MYSQL__SSL__ENABLE=on
+            EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem
+            EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem
+            EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem
+          EOF
      - name: setup
        env:
          MYSQL_TAG: ${{ matrix.mysql_tag }}
        if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d
+          echo EMQX_AUTH__MYSQL__SSL__ENABLE=off >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv4'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql)
-          sed -i "/auth.mysql.server/c auth.mysql.server = \"$server_address:3306\"" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv6'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql)
-          sed -i "/auth.mysql.server/c auth.mysql.server = \"$server_address:3306\"" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-      - name: setup
-        run: |
-          sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
      - name: run test cases
        run: |
+          export EMQX_AUTH__MYSQL__USERNAME=root \
+                 EMQX_AUTH__MYSQL__PASSWORD=public \
+                 EMQX_AUTH__MYSQL__DATABASE=mqtt \
+                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
          docker exec -i erlang sh -c "make ensure-rebar3"
          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_mysql"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
      - uses: actions/upload-artifact@v1
        if: failure()
        with:
@ -194,35 +200,35 @@ jobs:
        run: |
          docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache
          docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.ssl.enable[ \t]*=.*|auth.pgsql.ssl.enable = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__PGSQL__SSL__ENABLE=on
+          EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt
+          EOF
      - name: setup
        env:
          PGSQL_TAG: ${{ matrix.pgsql_tag }}
        if: matrix.connect_type == 'tcp'
        run: |
          docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo EMQX_AUTH__PGSQL__SSL__ENABLE=off >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv4'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql)
-          sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = \"$server_address:5432\"|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
      - name: setup
        if: matrix.network_type == 'ipv6'
        run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql)
-          sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = \"$server_address:5432\"|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
      - name: run test cases
        run: |
+          export EMQX_AUTH__PGSQL__USERNAME=root \
+                 EMQX_AUTH__PGSQL__PASSWORD=public \
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
          docker exec -i erlang sh -c "make ensure-rebar3"
          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_pgsql"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
      - uses: actions/upload-artifact@v1
        if: failure()
        with:
@ -257,51 +263,69 @@ jobs:
        run: |
          set -exu
          docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.enable[[:space:]]*=.*|auth.redis.ssl.enable = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.cacertfile[[:space:]]*=.*|auth.redis.ssl.cacertfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.certfile[[:space:]]*=.*|auth.redis.ssl.certfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.keyfile[[:space:]]*=.*|auth.redis.ssl.keyfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__SSL__ENABLE=on
+          EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt
+          EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt
+          EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key
+          EOF
      - name: setup
        env:
          REDIS_TAG: ${{ matrix.redis_tag }}
        if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d
+          echo EMQX_AUTH__REDIS__SSL__ENABLE=off >> "$GITHUB_ENV"
      - name: get server address
        if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
        run: |
          set -exu
          ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis)
          ipv6_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' redis)
-          echo "redis_ipv4_address=$ipv4_address" >> $GITHUB_ENV
-          echo "redis_ipv6_address=$ipv6_address" >> $GITHUB_ENV
+          cat <<-EOF >> "$GITHUB_ENV"
+          redis_ipv4_address=$ipv4_address
+          redis_ipv6_address=$ipv6_address
+          EOF
      - name: setup
        if: matrix.node_type == 'single' && matrix.connect_type == 'tcp'
        run: |
          set -exu
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:6379\"|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=single
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6379
+          EOF
      - name: setup
        if: matrix.node_type == 'single' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
        run: |
          set -exu
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:6380\"|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=single
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6380
+          EOF
      - name: setup
        if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp'
        run: |
          set -exu
-          sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002\"|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=cluster
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:7000
+          EOF
      - name: setup
        if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
        run: |
          set -exu
-          sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002\"|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=cluster
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000
+          EOF
      - name: run test cases
        if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
        run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
          docker exec -i erlang sh -c "make ensure-rebar3"
          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_redis"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
      - uses: actions/upload-artifact@v1
        if: failure()
        with:
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@ -29,25 +29,24 @@ jobs:
          run: |
            docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache
            docker-compose -f .ci/apps_tests/docker-compose.yaml up -d
-        - name: set config files
-          run: |
-            sed -i 's|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = "ldap_server"|g' apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
-            sed -i 's|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = "mongo_server:27017"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-            sed -i 's|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = "redis_server:6379"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf

-            sed -i 's|^[#[:space:]]*auth.mysql.server[[:space:]]*=.*|auth.mysql.server = "mysql_server:3306"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-
-            sed -i 's|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = "pgsql_server:5432"|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.username[[:space:]]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.password[[:space:]]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.database[[:space:]]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
        - name: run tests
          run: |
+            export EMQX_AUTH__LDAP__SERVERS=ldap_server \
+                   EMQX_AUTH__MONGO__SERVER=mongo_server:27017 \
+                   EMQX_AUTH__REDIS__SERVER=redis_server:6379 \
+                   EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 \
+                   EMQX_AUTH__MYSQL__USERNAME=root \
+                   EMQX_AUTH__MYSQL__PASSWORD=public \
+                   EMQX_AUTH__MYSQL__DATABASE=mqtt \
+                   EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432 \
+                   EMQX_AUTH__PGSQL__USERNAME=root \
+                   EMQX_AUTH__PGSQL__PASSWORD=public \
+                   EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                   CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+            printenv > .env
            docker exec -i erlang bash -c "make xref"
-            docker exec -i erlang bash -c "make ct"
+            docker exec --env-file .env -i erlang bash -c "make ct"
            docker exec -i erlang bash -c "make cover"
            docker exec -i erlang bash -c "make coveralls"
        - uses: actions/upload-artifact@v1
--- a/bin/emqx
+++ b/bin/emqx
@ -20,6 +20,9 @@ mkdir -p "$RUNNER_LOG_DIR"
 # Make sure data directory exists
 mkdir -p "$RUNNER_DATA_DIR"

+# cuttlefish try to read environment variables starting with "EMQX_", if not specified
+export CUTTLEFISH_ENV_OVERRIDE_PREFIX="${CUTTLEFISH_ENV_OVERRIDE_PREFIX:-EMQX_}"
+
 relx_usage() {
    command="$1"

--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@ -42,7 +42,7 @@ initContainers: {}

 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
-  EMQX_CLUSTER__K8S__APISERVER: \"https://kubernetes.default.svc:443\"
+  EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
  ## The address type is used to extract host from k8s service.
  ## Value: ip | dns | hostname
  ## Note：Hostname is only supported after v4.0-rc.2
--- a/deploy/docker/README.md
+++ b/deploy/docker/README.md
@ -41,11 +41,9 @@ The emqx broker runs as linux user `emqx` in the docker container.

 Use the environment variable to configure the EMQ X docker container.

-The environment variables which with ``EMQX_`` prefix are mapped to configuration fils.
+By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files.

-+ Prefix ``EMQX_`` is removed
-+ All upper case letters is replaced with lower case letters
-+ ``__`` is replaced with ``.``
+You can change the prefix by overriding "CUTTLEFISH_ENV_OVERRIDE_PREFIX".

 Example:

@ -54,6 +52,17 @@ EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
 EMQX_MQTT__MAX_PACKET_SIZE              <--> mqtt.max_packet_size
 ```

+ Prefix ``EMQX_`` is removed
+ All upper case letters is replaced with lower case letters
+ ``__`` is replaced with ``.``
+
+If `CUTTLEFISH_ENV_OVERRIDE_PREFIX=DEV_` is set:
+
+```bash
+DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
+DEV_MQTT__MAX_PACKET_SIZE              <--> mqtt.max_packet_size
+```
+
 Non mapped environment variables:

 ```bash
@ -189,16 +198,6 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \
    emqx/emqx:latest
 ```

-#### Mask Sensitive Configuration
-
-Use ``MASK_CONFIG_FILTER`` to hide senstive configuration values from leaking to logging system.
-
-For example, set ``MASK_CONFIG_FILTER="password|token"`` to hide all configuration names containing those keywords.
-
-By default emqx masks the configuration using following filter `"password|passwd|key|token|secret"`. Setting ``MASK_CONFIG_FILTER`` will be merged with the default filter.
-
-The configuration should match whole word (after splitting it by '.') with `MASK_CONFIG_FILTER`. You can use commas, spaces or other required separators to separate different words.
-
 ### Cluster

 EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details.
--- a/deploy/docker/docker-entrypoint.sh
+++ b/deploy/docker/docker-entrypoint.sh
@ -90,79 +90,6 @@ if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS" ]]; then
    export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400
 fi

-# Fix issue #42 - export env EMQX_DASHBOARD__DEFAULT_USER__PASSWORD to configure
-# 'dashboard.default_user.password' in etc/plugins/emqx_dashboard.conf
-if [[ -n "$EMQX_ADMIN_PASSWORD" ]]; then
-    export EMQX_DASHBOARD__DEFAULT_USER__PASSWORD=$EMQX_ADMIN_PASSWORD
-fi
-
-# echo value of $VAR hiding secrets if any
-# SYNOPSIS
-#     echo_value KEY VALUE
-echo_value() {
-    # get MASK_CONFIG
-    MASK_CONFIG_FILTER="$MASK_CONFIG_FILTER|password|passwd|key|token|secret"
-    FORMAT_MASK_CONFIG_FILTER=$(echo "$MASK_CONFIG_FILTER" | sed -r -e 's/^[^A-Za-z0-9_]+//' -e 's/[^A-Za-z0-9_]+$//' -e 's/[^A-Za-z0-9_]+/|/g')
-    local key=$1
-    local value=$2
-    # check if contains sensitive value
-    if echo "$key" | grep -iqwE "$FORMAT_MASK_CONFIG_FILTER"; then
-        echo "$key=***secret***"
-    else
-        echo "$key=$value"
-    fi
-}
-
-# fill config on specific file if the key exists
-# SYNOPSIS
-#     try_fill_config FILE KEY VALUE
-try_fill_config() {
-    local file=$1
-    local key=$2
-    local value=$3
-    local escaped_key
-    # shellcheck disable=SC2001
-    escaped_key=$(echo "$key" | sed 's/[^a-zA-Z0-9_]/\\&/g')
-    local escaped_value
-    escaped_value=$(echo "$value" | sed 's/[\/&]/\\&/g')
-    if grep -qE "^[#[:space:]]*$escaped_key\s*=" "$file"; then
-        echo_value "$key" "$value"
-        if [[ -z "$value" ]]; then
-            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file" > tmpfile && cat tmpfile > "$file"
-        else
-            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file" > tmpfile && cat tmpfile > "$file"
-        fi
-    # Check if config has a numbering system, but no existing configuration line in file
-    elif echo "$key" | grep -qE '\.\d+|\d+\.'; then
-        if [[ -n "$value" ]]; then
-            local template
-            template="$(echo "$escaped_key" | sed -r -e 's/\\\.[0-9]+/\\.[0-9]+/g' -e 's/[0-9]+\\\./[0-9]+\\./g')"
-            if grep -qE "^[#[:space:]]*$template\s*=" "$file"; then
-                echo_value "$key" "$value"
-                sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
-                echo "$key = $value" >> "$file"
-            fi
-        fi
-    fi
-}
-
-# Catch all EMQX_ prefix environment variable and match it in configure file
-CONFIG_FILE="$_EMQX_HOME/etc/emqx.conf"
-CONFIG_PLUGINS="$_EMQX_HOME/etc/plugins"
-for VAR in $(compgen -e); do
-    # Config normal keys such like node.name = emqx@127.0.0.1
-    if echo "$VAR" | grep -q '^EMQX_'; then
-        VAR_NAME=$(echo "$VAR" | sed -e 's/^EMQX_//' -e 's/__/./g' | tr '[:upper:]' '[:lower:]' | tr -d '[:cntrl:]')
-        VAR_VALUE=$(echo "${!VAR}" | tr -d '[:cntrl:]')
-        # Config in emqx.conf
-        try_fill_config "$CONFIG_FILE" "$VAR_NAME" "$VAR_VALUE"
-        # Config in plugins/*
-        for CONFIG_PLUGINS_FILE in "$CONFIG_PLUGINS"/*; do
-            try_fill_config "$CONFIG_PLUGINS_FILE" "$VAR_NAME" "$VAR_VALUE"
-        done
-    fi
-done
-
 # fill tuples on specific file
 # SYNOPSIS
 #     fill_tuples FILE [ELEMENTS ...]
--- a/etc/emqx.conf
+++ b/etc/emqx.conf
@ -1088,10 +1088,18 @@ listener.tcp.external.access.1 = "allow all"

 ## Enable the option for X.509 certificate based authentication.
 ## EMQX will use the common name of certificate as MQTT username.
+## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT.
 ##
-## Value: cn | dn | crt
+## Value: cn | dn | crt | pem | md5
 ## listener.tcp.external.peer_cert_as_username = cn

+## Enable the option for X.509 certificate based authentication.
+## EMQX will use the common name of certificate as MQTT clientid.
+## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT.
+##
+## Value: cn | dn | crt | pem | md5
+## listener.tcp.external.peer_cert_as_clientid = cn
+
 ## The TCP backlog defines the maximum length that the queue of pending
 ## connections can grow to.
 ##
@ -1442,10 +1450,18 @@ listener.ssl.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,T

 ## Use the CN, DN or CRT field from the client certificate as a username.
 ## Notice that 'verify' should be set as 'verify_peer'.
+## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT.
 ##
-## Value: cn | dn | crt
+## Value: cn | dn | crt | pem | md5
 ## listener.ssl.external.peer_cert_as_username = cn

+## Use the CN, DN or CRT field from the client certificate as a username.
+## Notice that 'verify' should be set as 'verify_peer'.
+## 'pem' encodes CRT in base64, and md5 is the md5 hash of CRT.
+##
+## Value: cn | dn | crt | pem | md5
+## listener.ssl.external.peer_cert_as_clientid = cn
+
 ## TCP backlog for the SSL connection.
 ##
 ## See listener.tcp.$name.backlog
@ -1889,9 +1905,14 @@ listener.wss.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,T

 ## See: listener.ssl.$name.peer_cert_as_username
 ##
-## Value: cn | dn | crt
+## Value: cn | dn | crt | pem | md5
 ## listener.wss.external.peer_cert_as_username = cn

+## See: listener.ssl.$name.peer_cert_as_clientid
+##
+## Value: cn | dn | crt | pem | md5
+## listener.wss.external.peer_cert_as_clientid = cn
+
 ## TCP backlog for the WebSocket/SSL connection.
 ##
 ## See: listener.tcp.$name.backlog
--- a/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema
+++ b/lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema
@ -6,7 +6,8 @@
 ]}.

 {mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [
-  {datatype, string}
+  {datatype, string},
+  {override_env, "ADMIN_PASSWORD"}
 ]}.

 {mapping, "dashboard.listener.http.port", "emqx_dashboard.listeners", [
--- a/priv/emqx.schema
+++ b/priv/emqx.schema
@ -202,7 +202,8 @@ end}.

 %% @doc Node name
 {mapping, "node.name", "vm_args.-name", [
-  {default, "emqx@127.0.0.1"}
+  {default, "emqx@127.0.0.1"},
+  {override_env, "NODE_NAME"}
 ]}.

 %% @doc Specify SSL Options in the file if using SSL for erlang distribution
@ -1211,7 +1212,11 @@ end}.
 ]}.

 {mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [
-  {datatype, {enum, [cn, dn, crt]}}
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
+]}.
+
+{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
 ]}.

 {mapping, "listener.tcp.$name.backlog", "emqx.listeners", [
@ -1425,7 +1430,11 @@ end}.
 ]}.

 {mapping, "listener.ssl.$name.peer_cert_as_username", "emqx.listeners", [
-  {datatype, {enum, [cn, dn, crt]}}
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
+]}.
+
+{mapping, "listener.ssl.$name.peer_cert_as_clientid", "emqx.listeners", [
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
 ]}.

 %%--------------------------------------------------------------------
@ -1765,7 +1774,11 @@ end}.
 ]}.

 {mapping, "listener.wss.$name.peer_cert_as_username", "emqx.listeners", [
-  {datatype, {enum, [cn, dn, crt]}}
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
+]}.
+
+{mapping, "listener.wss.$name.peer_cert_as_clientid", "emqx.listeners", [
+  {datatype, {enum, [cn, dn, crt, pem, md5]}}
 ]}.

 {mapping, "listener.wss.$name.compress", "emqx.listeners", [
@ -1904,6 +1917,7 @@ end}.
                          {fail_if_no_subprotocol, cuttlefish:conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf, undefined)},
                          {supported_subprotocols, string:tokens(cuttlefish:conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")},
                          {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)},
+                          {peer_cert_as_clientid, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_clientid", Conf, undefined)},
                          {compress, cuttlefish:conf_get(Prefix ++ ".compress", Conf, undefined)},
                          {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)},
                          {max_frame_size, cuttlefish:conf_get(Prefix ++ ".max_frame_size", Conf, undefined)},
--- a/src/emqx_channel.erl
+++ b/src/emqx_channel.erl
@ -220,13 +220,19 @@ setting_peercert_infos(NoSSL, ClientInfo, _Options)
 setting_peercert_infos(Peercert, ClientInfo, Options) ->
    {DN, CN} = {esockd_peercert:subject(Peercert),
                esockd_peercert:common_name(Peercert)},
-    Username = case proplists:get_value(peer_cert_as_username, Options) of
+    Username = peer_cert_as(peer_cert_as_username, Options, Peercert, DN, CN),
+    ClientId = peer_cert_as(peer_cert_as_clientid, Options, Peercert, DN, CN),
+    ClientInfo#{username => Username, clientid => ClientId, dn => DN, cn => CN}.
+
+peer_cert_as(Key, Options, Peercert, DN, CN) ->
+    case proplists:get_value(Key, Options) of
         cn  -> CN;
         dn  -> DN;
         crt -> Peercert;
+         pem -> base64:encode(Peercert);
+         md5 -> emqx_passwd:hash(md5, Peercert);
         _   -> undefined
-               end,
-    ClientInfo#{username => Username, dn => DN, cn => CN}.
+     end.

 take_ws_cookie(ClientInfo, ConnInfo) ->
    case maps:take(ws_cookie, ConnInfo) of
--- a/src/emqx_shared_sub.erl
+++ b/src/emqx_shared_sub.erl
@ -282,7 +282,7 @@ do_pick_subscriber(_Group, _Topic, hash_topic, _ClientId, SourceTopic, Count) ->
    1 + erlang:phash2(SourceTopic) rem Count;
 do_pick_subscriber(Group, Topic, round_robin, _ClientId, _SourceTopic, Count) ->
    Rem = case erlang:get({shared_sub_round_robin, Group, Topic}) of
-              undefined -> 0;
+              undefined -> rand:uniform(Count) - 1;
              N -> (N + 1) rem Count
          end,
    _ = erlang:put({shared_sub_round_robin, Group, Topic}, Rem),