yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4195 from z8674558/env-override 

feat(bin): let cuttlefish read env var
This commit is contained in:
Zaiming Shi 2021-02-18 22:54:00 +01:00 committed by GitHub
parent d2bd9b4078 2b9a702b70
commit dfcc2dd566
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 121 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.ci/build_packages/tests.sh
View File

@ -23,8 +23,8 @@ emqx_test(){
"zip") "zip")
packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
unzip -q "${PACKAGE_PATH}/${packagename}" unzip -q "${PACKAGE_PATH}/${packagename}"
sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" "${PACKAGE_PATH}"/emqx/etc/emqx.conf export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" "${PACKAGE_PATH}"/emqx/etc/emqx.conf EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
echo "running ${packagename} start" echo "running ${packagename} start"
@ -96,8 +96,8 @@ emqx_test(){
} }
running_test(){ running_test(){
sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" /etc/emqx/emqx.conf export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" /etc/emqx/emqx.conf EMQX_MQTT__MAX_TOPIC_ALIAS=10
sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
emqx start || tail /var/log/emqx/erlang.log.1 emqx start || tail /var/log/emqx/erlang.log.1

138
.github/workflows/run_cts_tests.yaml vendored
View File

@ -36,18 +36,18 @@ jobs:
- name: setup - name: setup
if: matrix.network_type == 'ipv4' if: matrix.network_type == 'ipv4'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
- name: setup - name: setup
if: matrix.network_type == 'ipv6' if: matrix.network_type == 'ipv6'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
- name: run test cases - name: run test cases
run: | run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3" docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap" docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
docker exec -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: failure() if: failure()
with: with:
@ -78,30 +78,34 @@ jobs:
if: matrix.connect_type == 'tls' if: matrix.connect_type == 'tls'
run: | run: |
docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d
sed -i 's|^[#[:space:]]*auth.mongo.ssl[[:space:]]*=.*|auth.mongo.ssl = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf EMQX_AUTH__MONGO__SSL=on
sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf EMQX_AUTH__MONGO__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem
sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf EMQX_AUTH__MONGO__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem
EMQX_AUTH__MONGO__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem
EOF
- name: setup - name: setup
env: env:
MONGO_TAG: ${{ matrix.mongo_tag }} MONGO_TAG: ${{ matrix.mongo_tag }}
if: matrix.connect_type == 'tcp' if: matrix.connect_type == 'tcp'
run: docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d run: |
docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d
echo EMQX_AUTH__MONGO__SSL=off >> "$GITHUB_ENV"
- name: setup - name: setup
if: matrix.network_type == 'ipv4' if: matrix.network_type == 'ipv4'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
- name: setup - name: setup
if: matrix.network_type == 'ipv6' if: matrix.network_type == 'ipv6'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo) echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
- name: run test cases - name: run test cases
run: | run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3" docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo" docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
docker exec -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: failure() if: failure()
with: with:
@ -132,35 +136,37 @@ jobs:
if: matrix.connect_type == 'tls' if: matrix.connect_type == 'tls'
run: | run: |
docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d
sed -i 's|^[#[:space:]]*auth.mysql.ssl[[:space:]]*=.*|auth.mysql.ssl = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i 's|^[#[:space:]]*auth.mysql.ssl.cacertfile[[:space:]]*=.*|auth.mysql.ssl.cacertfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf EMQX_AUTH__MYSQL__SSL=on
sed -i 's|^[#[:space:]]*auth.mysql.ssl.certfile[[:space:]]*=.*|auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem
sed -i 's|^[#[:space:]]*auth.mysql.ssl.keyfile[[:space:]]*=.*|auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem
EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem
EOF
- name: setup - name: setup
env: env:
MYSQL_TAG: ${{ matrix.mysql_tag }} MYSQL_TAG: ${{ matrix.mysql_tag }}
if: matrix.connect_type == 'tcp' if: matrix.connect_type == 'tcp'
run: docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d run: |
docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d
echo EMQX_AUTH__MYSQL__SSL=off >> "$GITHUB_ENV"
- name: setup - name: setup
if: matrix.network_type == 'ipv4' if: matrix.network_type == 'ipv4'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
- name: setup - name: setup
if: matrix.network_type == 'ipv6' if: matrix.network_type == 'ipv6'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql) echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
- name: setup
run: |
sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
- name: run test cases - name: run test cases
run: | run: |
export EMQX_AUTH__MYSQL__USERNAME=root \
EMQX_AUTH__MYSQL__PASSWORD=public \
EMQX_AUTH__MYSQL__DATABASE=mqtt \
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3" docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql" docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
docker exec -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: failure() if: failure()
with: with:
@ -194,35 +200,35 @@ jobs:
run: | run: |
docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache
docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d
sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf EMQX_AUTH__PGSQL__SSL=on
sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt
sed -i 's|^[#[:space:]]*auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf EOF
sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
- name: setup - name: setup
env: env:
PGSQL_TAG: ${{ matrix.pgsql_tag }} PGSQL_TAG: ${{ matrix.pgsql_tag }}
if: matrix.connect_type == 'tcp' if: matrix.connect_type == 'tcp'
run: | run: |
docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d
sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf echo EMQX_AUTH__PGSQL__SSL=off >> "$GITHUB_ENV"
sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
- name: setup - name: setup
if: matrix.network_type == 'ipv4' if: matrix.network_type == 'ipv4'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
- name: setup - name: setup
if: matrix.network_type == 'ipv6' if: matrix.network_type == 'ipv6'
run: | run: |
server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql) echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
- name: run test cases - name: run test cases
run: | run: |
export EMQX_AUTH__PGSQL__USERNAME=root \
EMQX_AUTH__PGSQL__PASSWORD=public \
EMQX_AUTH__PGSQL__DATABASE=mqtt \
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3" docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql" docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
docker exec -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: failure() if: failure()
with: with:
@ -257,51 +263,69 @@ jobs:
run: | run: |
set -exu set -exu
docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d
sed -i 's|^[#[:space:]]*auth.redis.ssl[[:space:]]*=.*|auth.redis.ssl = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i 's|^[#[:space:]]*auth.redis.ssl.cacertfile[[:space:]]*=.*|auth.redis.ssl.cacertfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf EMQX_AUTH__REDIS__SSL=on
sed -i 's|^[#[:space:]]*auth.redis.ssl.certfile[[:space:]]*=.*|auth.redis.ssl.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt
sed -i 's|^[#[:space:]]*auth.redis.ssl.keyfile[[:space:]]*=.*|auth.redis.ssl.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt
EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key
EOF
- name: setup - name: setup
env: env:
REDIS_TAG: ${{ matrix.redis_tag }} REDIS_TAG: ${{ matrix.redis_tag }}
if: matrix.connect_type == 'tcp' if: matrix.connect_type == 'tcp'
run: docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d run: |
docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d
echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV"
- name: get server address - name: get server address
if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
run: | run: |
set -exu set -exu
ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis) ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis)
ipv6_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' redis) ipv6_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' redis)
echo "redis_ipv4_address=$ipv4_address" >> $GITHUB_ENV cat <<-EOF >> "$GITHUB_ENV"
echo "redis_ipv6_address=$ipv6_address" >> $GITHUB_ENV redis_ipv4_address=$ipv4_address
redis_ipv6_address=$ipv6_address
EOF
- name: setup - name: setup
if: matrix.node_type == 'single' && matrix.connect_type == 'tcp' if: matrix.node_type == 'single' && matrix.connect_type == 'tcp'
run: | run: |
set -exu set -exu
sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6379|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf cat <<-EOF >> "$GITHUB_ENV"
EMQX_AUTH__REDIS__TYPE=single
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6379
EOF
- name: setup - name: setup
if: matrix.node_type == 'single' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' if: matrix.node_type == 'single' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
run: | run: |
set -exu set -exu
sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6380|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf cat <<-EOF >> "$GITHUB_ENV"
EMQX_AUTH__REDIS__TYPE=single
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6380
EOF
- name: setup - name: setup
if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp' if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp'
run: | run: |
set -exu set -exu
sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf EMQX_AUTH__REDIS__TYPE=cluster
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:7000
EOF
- name: setup - name: setup
if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
run: | run: |
set -exu set -exu
sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf cat <<-EOF >> "$GITHUB_ENV"
sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf EMQX_AUTH__REDIS__TYPE=cluster
EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000
EOF
- name: run test cases - name: run test cases
if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
run: | run: |
export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang sh -c "make ensure-rebar3" docker exec -i erlang sh -c "make ensure-rebar3"
docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis" docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
docker exec -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis" docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: failure() if: failure()
with: with:

29
.github/workflows/run_test_cases.yaml vendored
View File

@ -29,25 +29,24 @@ jobs:
run: | run: |
docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache
docker-compose -f .ci/apps_tests/docker-compose.yaml up -d docker-compose -f .ci/apps_tests/docker-compose.yaml up -d
- name: set config files
run: |
sed -i 's|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = ldap_server|g' apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
sed -i 's|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = mongo_server:27017|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
sed -i 's|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = redis_server:6379|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
sed -i 's|^[#[:space:]]*auth.mysql.server[[:space:]]*=.*|auth.mysql.server = mysql_server:3306|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
sed -i 's|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = pgsql_server:5432|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
sed -i 's|^[#[:space:]]*auth.pgsql.username[[:space:]]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
sed -i 's|^[#[:space:]]*auth.pgsql.password[[:space:]]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
sed -i 's|^[#[:space:]]*auth.pgsql.database[[:space:]]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
- name: run tests - name: run tests
run: | run: |
export EMQX_AUTH__LDAP__SERVERS=ldap_server \
EMQX_AUTH__MONGO__SERVER=mongo_server:27017 \
EMQX_AUTH__REDIS__SERVER=redis_server:6379 \
EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 \
EMQX_AUTH__MYSQL__USERNAME=root \
EMQX_AUTH__MYSQL__PASSWORD=public \
EMQX_AUTH__MYSQL__DATABASE=mqtt \
EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432 \
EMQX_AUTH__PGSQL__USERNAME=root \
EMQX_AUTH__PGSQL__PASSWORD=public \
EMQX_AUTH__PGSQL__DATABASE=mqtt \
CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
printenv > .env
docker exec -i erlang bash -c "make xref" docker exec -i erlang bash -c "make xref"
docker exec -i erlang bash -c "make ct" docker exec --env-file .env -i erlang bash -c "make ct"
docker exec -i erlang bash -c "make cover" docker exec -i erlang bash -c "make cover"
docker exec -i erlang bash -c "make coveralls" docker exec -i erlang bash -c "make coveralls"
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1

3
bin/emqx
View File

@ -20,6 +20,9 @@ mkdir -p "$RUNNER_LOG_DIR"
# Make sure data directory exists # Make sure data directory exists
mkdir -p "$RUNNER_DATA_DIR" mkdir -p "$RUNNER_DATA_DIR"
# cuttlefish try to read environment variables starting with "EMQX_", if not specified
export CUTTLEFISH_ENV_OVERRIDE_PREFIX="${CUTTLEFISH_ENV_OVERRIDE_PREFIX:-EMQX_}"
relx_usage() { relx_usage() {
command="$1" command="$1"

29
deploy/docker/README.md
View File

@ -41,11 +41,9 @@ The emqx broker runs as linux user `emqx` in the docker container.
Use the environment variable to configure the EMQ X docker container. Use the environment variable to configure the EMQ X docker container.
The environment variables which with ``EMQX_`` prefix are mapped to configuration fils. By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files.
+ Prefix ``EMQX_`` is removed You can change the prefix by overriding "CUTTLEFISH_ENV_OVERRIDE_PREFIX".
+ All upper case letters is replaced with lower case letters
+ ``__`` is replaced with ``.``
Example: Example:
@ -54,6 +52,17 @@ EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
EMQX_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size EMQX_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size
``` ```
+ Prefix ``EMQX_`` is removed
+ All upper case letters is replaced with lower case letters
+ ``__`` is replaced with ``.``
If `CUTTLEFISH_ENV_OVERRIDE_PREFIX=DEV_` is set:
```bash
DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
DEV_MQTT__MAX_PACKET_SIZE <--> mqtt.max_packet_size
```
Non mapped environment variables: Non mapped environment variables:
```bash ```bash
@ -189,16 +198,6 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \
emqx/emqx:latest emqx/emqx:latest
``` ```
#### Mask Sensitive Configuration
Use ``MASK_CONFIG_FILTER`` to hide senstive configuration values from leaking to logging system.
For example, set ``MASK_CONFIG_FILTER="password|token"`` to hide all configuration names containing those keywords.
By default emqx masks the configuration using following filter `"password|passwd|key|token|secret"`. Setting ``MASK_CONFIG_FILTER`` will be merged with the default filter.
The configuration should match whole word (after splitting it by '.') with `MASK_CONFIG_FILTER`. You can use commas, spaces or other required separators to separate different words.
### Cluster ### Cluster
EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details. EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details.
@ -234,7 +233,7 @@ Let's create a static node list cluster from docker-compose.
emqx-bridge: emqx-bridge:
aliases: aliases:
- node2.emqx.io - node2.emqx.io
networks: networks:
emqx-bridge: emqx-bridge:
driver: bridge driver: bridge

73
deploy/docker/docker-entrypoint.sh
View File

@ -90,79 +90,6 @@ if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS" ]]; then
export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400 export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400
fi fi
# Fix issue #42 - export env EMQX_DASHBOARD__DEFAULT_USER__PASSWORD to configure
# 'dashboard.default_user.password' in etc/plugins/emqx_dashboard.conf
if [[ -n "$EMQX_ADMIN_PASSWORD" ]]; then
export EMQX_DASHBOARD__DEFAULT_USER__PASSWORD=$EMQX_ADMIN_PASSWORD
fi
# echo value of $VAR hiding secrets if any
# SYNOPSIS
# echo_value KEY VALUE
echo_value() {
# get MASK_CONFIG
MASK_CONFIG_FILTER="$MASK_CONFIG_FILTER|password|passwd|key|token|secret"
FORMAT_MASK_CONFIG_FILTER=$(echo "$MASK_CONFIG_FILTER" | sed -r -e 's/^[^A-Za-z0-9_]+//' -e 's/[^A-Za-z0-9_]+$//' -e 's/[^A-Za-z0-9_]+/|/g')
local key=$1
local value=$2
# check if contains sensitive value
if echo "$key" | grep -iqwE "$FORMAT_MASK_CONFIG_FILTER"; then
echo "$key=***secret***"
else
echo "$key=$value"
fi
}
# fill config on specific file if the key exists
# SYNOPSIS
# try_fill_config FILE KEY VALUE
try_fill_config() {
local file=$1
local key=$2
local value=$3
local escaped_key
# shellcheck disable=SC2001
escaped_key=$(echo "$key" | sed 's/[^a-zA-Z0-9_]/\\&/g')
local escaped_value
escaped_value=$(echo "$value" | sed 's/[\/&]/\\&/g')
if grep -qE "^[#[:space:]]*$escaped_key\s*=" "$file"; then
echo_value "$key" "$value"
if [[ -z "$value" ]]; then
sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file" > tmpfile && cat tmpfile > "$file"
else
sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file" > tmpfile && cat tmpfile > "$file"
fi
# Check if config has a numbering system, but no existing configuration line in file
elif echo "$key" | grep -qE '\.\d+|\d+\.'; then
if [[ -n "$value" ]]; then
local template
template="$(echo "$escaped_key" | sed -r -e 's/\\\.[0-9]+/\\.[0-9]+/g' -e 's/[0-9]+\\\./[0-9]+\\./g')"
if grep -qE "^[#[:space:]]*$template\s*=" "$file"; then
echo_value "$key" "$value"
sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
echo "$key = $value" >> "$file"
fi
fi
fi
}
# Catch all EMQX_ prefix environment variable and match it in configure file
CONFIG_FILE="$_EMQX_HOME/etc/emqx.conf"
CONFIG_PLUGINS="$_EMQX_HOME/etc/plugins"
for VAR in $(compgen -e); do
# Config normal keys such like node.name = emqx@127.0.0.1
if echo "$VAR" | grep -q '^EMQX_'; then
VAR_NAME=$(echo "$VAR" | sed -e 's/^EMQX_//' -e 's/__/./g' | tr '[:upper:]' '[:lower:]' | tr -d '[:cntrl:]')
VAR_VALUE=$(echo "${!VAR}" | tr -d '[:cntrl:]')
# Config in emqx.conf
try_fill_config "$CONFIG_FILE" "$VAR_NAME" "$VAR_VALUE"
# Config in plugins/*
for CONFIG_PLUGINS_FILE in "$CONFIG_PLUGINS"/*; do
try_fill_config "$CONFIG_PLUGINS_FILE" "$VAR_NAME" "$VAR_VALUE"
done
fi
done
# fill tuples on specific file # fill tuples on specific file
# SYNOPSIS # SYNOPSIS
# fill_tuples FILE [ELEMENTS ...] # fill_tuples FILE [ELEMENTS ...]

3
lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema
View File

@ -6,7 +6,8 @@
]}. ]}.
{mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [ {mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [
{datatype, string} {datatype, string},
{override_env, "ADMIN_PASSWORD"}
]}. ]}.
{mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ {mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [

3
priv/emqx.schema
View File

@ -202,7 +202,8 @@ end}.
%% @doc Node name %% @doc Node name
{mapping, "node.name", "vm_args.-name", [ {mapping, "node.name", "vm_args.-name", [
{default, "emqx@127.0.0.1"} {default, "emqx@127.0.0.1"},
{override_env, "NODE_NAME"}
]}. ]}.
%% @doc Specify SSL Options in the file if using SSL for erlang distribution %% @doc Specify SSL Options in the file if using SSL for erlang distribution

2
rebar.config
View File

@ -46,7 +46,7 @@
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}}
, {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.1.0"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}}
, {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.1"}}}