From 455f210b4cb4ba6e09fe342061019cafdd2ce67f Mon Sep 17 00:00:00 2001 From: Yudai Kiyofuji <57182816+z8674558@users.noreply.github.com> Date: Tue, 19 Jan 2021 19:52:17 +0900 Subject: [PATCH 01/58] feat(cuttlefish): use hocon as a parsing function (#4008) * chore(conf): add quotation for hocon * chore(conf): fix paths incompatible with hocon * chore(conf): use hocon as parsing function * chore(docker): add quotation to some env variables for hocon --- .ci/fvt_tests/docker-compose.yaml | 14 +- .github/workflows/run_cts_tests.yaml | 62 +++---- .github/workflows/run_test_cases.yaml | 10 +- apps/emqx_auth_http/etc/emqx_auth_http.conf | 28 ++-- .../emqx_auth_http/priv/emqx_auth_http.schema | 14 +- apps/emqx_auth_http/rebar.config | 2 +- apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf | 8 +- apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema | 6 +- apps/emqx_auth_jwt/rebar.config | 2 +- apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf | 16 +- .../emqx_auth_ldap/priv/emqx_auth_ldap.schema | 4 +- apps/emqx_auth_ldap/rebar.config | 2 +- .../etc/emqx_auth_mnesia.conf | 12 +- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 28 ++-- .../priv/emqx_auth_mongo.schema | 4 +- apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf | 18 +- .../priv/emqx_auth_mysql.schema | 4 +- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 20 +-- .../priv/emqx_auth_pgsql.schema | 6 +- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 28 ++-- .../priv/emqx_auth_redis.schema | 4 +- .../etc/emqx_bridge_mqtt.conf | 28 ++-- apps/emqx_coap/etc/emqx_coap.conf | 24 +-- apps/emqx_coap/rebar.config | 4 +- apps/emqx_dashboard/etc/emqx_dashboard.conf | 16 +- .../emqx_dashboard/priv/emqx_dashboard.schema | 6 +- apps/emqx_exhook/etc/emqx_exhook.conf | 8 +- apps/emqx_exhook/rebar.config | 3 +- apps/emqx_exproto/etc/emqx_exproto.conf | 30 ++-- apps/emqx_exproto/priv/emqx_exproto.schema | 6 +- apps/emqx_exproto/rebar.config | 3 +- apps/emqx_lwm2m/etc/emqx_lwm2m.conf | 40 ++--- apps/emqx_lwm2m/rebar.config | 1 - apps/emqx_management/etc/emqx_management.conf | 14 +- .../priv/emqx_management.schema | 6 +- apps/emqx_prometheus/etc/emqx_prometheus.conf | 2 +- apps/emqx_psk_file/etc/emqx_psk_file.conf | 4 +- apps/emqx_retainer/etc/emqx_retainer.conf | 2 +- apps/emqx_retainer/rebar.config | 2 +- .../etc/emqx_rule_engine.conf | 2 +- .../test/emqx_rule_engine_SUITE.erl | 4 +- apps/emqx_sn/etc/emqx_sn.conf | 6 +- apps/emqx_sn/priv/emqx_sn.schema | 24 ++- apps/emqx_sn/rebar.config | 3 +- apps/emqx_stomp/etc/emqx_stomp.conf | 14 +- apps/emqx_stomp/priv/emqx_stomp.schema | 4 +- apps/emqx_telemetry/etc/emqx_telemetry.conf | 6 +- apps/emqx_web_hook/etc/emqx_web_hook.conf | 34 ++-- apps/emqx_web_hook/rebar.config | 3 +- deploy/charts/emqx/values.yaml | 6 +- deploy/docker/start.sh | 2 +- etc/emqx.conf | 157 +++++++++--------- priv/emqx.schema | 22 ++- rebar.config | 2 +- rebar.config.erl | 2 +- test/emqx_listeners_SUITE.erl | 8 +- .../emqx_mini_plugin/rebar.config | 8 - 57 files changed, 389 insertions(+), 409 deletions(-) diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml index 50e5efa16..74c8854da 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/fvt_tests/docker-compose.yaml @@ -8,11 +8,11 @@ services: - "EMQX_NAME=emqx" - "EMQX_HOST=node1.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" + - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\"" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - command: - - /bin/sh + command: + - /bin/sh - -c - | sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf @@ -26,7 +26,7 @@ services: emqx-bridge: aliases: - node1.emqx.io - + emqx2: container_name: node2.emqx.io image: emqx/emqx:build-alpine-amd64 @@ -34,11 +34,11 @@ services: - "EMQX_NAME=emqx" - "EMQX_HOST=node2.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" + - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\"" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - command: - - /bin/sh + command: + - /bin/sh - -c - | sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 60a841525..1e9e41626 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -39,12 +39,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) - sed -i "/auth.ldap.servers/c auth.ldap.servers = $server_address" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "/auth.ldap.servers/c auth.ldap.servers = \"$server_address\"" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) - sed -i "/auth.ldap.servers/c auth.ldap.servers = $server_address" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "/auth.ldap.servers/c auth.ldap.servers = \"$server_address\"" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -79,15 +79,15 @@ jobs: if: matrix.connect_type == 'tls' run: | docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d - echo 'auth.mongo.ssl = on' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - echo 'auth.mongo.ssl.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl.enable = on' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl.cacertfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl.certfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + echo 'auth.mongo.ssl.keyfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl = true' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - # echo 'auth.mongo.ssl_opts.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + # echo 'auth.mongo.ssl.enable = true' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + # echo 'auth.mongo.ssl_opts.cacertfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + # echo 'auth.mongo.ssl_opts.certfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + # echo 'auth.mongo.ssl_opts.keyfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem"' >> apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup env: MONGO_TAG: ${{ matrix.mongo_tag }} @@ -97,12 +97,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo) - sed -i "/auth.mongo.server/c auth.mongo.server = $server_address:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i "/auth.mongo.server/c auth.mongo.server = \"$server_address:27017\"" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo) - sed -i "/auth.mongo.server/c auth.mongo.server = $server_address:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i "/auth.mongo.server/c auth.mongo.server = \"$server_address:27017\"" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -139,9 +139,9 @@ jobs: docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_mysql.conf echo 'auth.mysql.ssl = on' >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.cafile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - echo "auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.cafile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem\"" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.certfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem\"" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + echo "auth.mysql.ssl.keyfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem"\" >> apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: setup env: MYSQL_TAG: ${{ matrix.mysql_tag }} @@ -151,12 +151,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) - sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i "/auth.mysql.server/c auth.mysql.server = \"$server_address:3306\"" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql) - sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i "/auth.mysql.server/c auth.mysql.server = \"$server_address:3306\"" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -196,9 +196,9 @@ jobs: docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_pgsql.conf echo 'auth.pgsql.ssl = true' >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.certfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - echo "auth.pgsql.ssl_opts.keyfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + echo "auth.pgsql.ssl_opts.cacertfile = \"/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem\"" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + echo "auth.pgsql.ssl_opts.certfile = \"/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem\"" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + echo "auth.pgsql.ssl_opts.keyfile = \"/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem\"" >> apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup env: PGSQL_TAG: ${{ matrix.pgsql_tag }} @@ -208,12 +208,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) - sed -i "/auth.pgsql.server/c auth.pgsql.server = $server_address:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "/auth.pgsql.server/c auth.pgsql.server = \"$server_address:5432\"" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql) - sed -i "/auth.pgsql.server/c auth.pgsql.server = $server_address:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "/auth.pgsql.server/c auth.pgsql.server = \"$server_address:5432\"" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -253,10 +253,10 @@ jobs: set -exu docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d echo '\n' >> apps/emqx_auth_mongo/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl = on' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.cafile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - echo 'auth.redis.ssl.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.ssl.enable = on' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.ssl.cafile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt"' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.ssl.certfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt"' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf + echo 'auth.redis.ssl.keyfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key"' >> apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup env: REDIS_TAG: ${{ matrix.redis_tag }} @@ -274,24 +274,24 @@ jobs: if: matrix.node_type == 'singer' && matrix.connect_type == 'tcp' run: | set -exu - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6379" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "/auth.redis.server/c auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:6379\"" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup if: matrix.node_type == 'singer' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' run: | set -exu - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6380" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "/auth.redis.server/c auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:6380\"" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp' run: | set -exu sed -i "/auth.redis.type/c auth.redis.type = cluster" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "/auth.redis.server/c auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002\"" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: setup - if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' + if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5' run: | set -exu sed -i "/auth.redis.type/c auth.redis.type = cluster" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.redis.server/c auth.redis.server = ${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "/auth.redis.server/c auth.redis.server = \"${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002\"" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - name: run test cases if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5') run: | diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 41d224f48..db0ccb740 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -33,11 +33,11 @@ jobs: docker-compose -f .ci/apps_tests/docker-compose.yaml up -d - name: set config files run: | - sed -i "/auth.mysql.server/c auth.mysql.server = mysql_server:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - sed -i "/auth.redis.server/c auth.redis.server = redis_server:6379" apps/emqx_auth_redis/etc/emqx_auth_redis.conf - sed -i "/auth.mongo.server/c auth.mongo.server = mongo_server:27017" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - sed -i "/auth.pgsql.server/c auth.pgsql.server = pgsql_server:5432" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i "/auth.ldap.servers/c auth.ldap.servers = ldap_server" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "/auth.mysql.server/c auth.mysql.server = \"mysql_server:3306\"" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i "/auth.redis.server/c auth.redis.server = \"redis_server:6379\"" apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i "/auth.mongo.server/c auth.mongo.server = \"mongo_server:27017\"" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i "/auth.pgsql.server/c auth.pgsql.server = \"pgsql_server:5432\"" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i "/auth.ldap.servers/c auth.ldap.servers = \"ldap_server\"" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: run tests run: | docker exec -i erlang bash -c "make xref" diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 56f76cf39..f32b5ddc5 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -9,8 +9,8 @@ ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/auth, https://[::1]:8991/mqtt/auth -auth.http.auth_req = http://127.0.0.1:8991/mqtt/auth +## Examples: "http://127.0.0.1:8991/mqtt/auth", "https://[::1]:8991/mqtt/auth" +auth.http.auth_req.endpoint = "http://127.0.0.1:8991/mqtt/auth" ## Value: post | get auth.http.auth_req.method = post @@ -31,7 +31,7 @@ auth.http.auth_req.content_type = x-www-form-urlencoded ## - %k: websocket cookie ## ## Value: Params -auth.http.auth_req.params = clientid=%c,username=%u,password=%P +auth.http.auth_req.params = "clientid=%c,username=%u,password=%P" ##-------------------------------------------------------------------- ## Superuser request. @@ -40,8 +40,8 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/superuser, https://[::1]:8991/mqtt/superuser -#auth.http.super_req = http://127.0.0.1:8991/mqtt/superuser +## Examples: "http://127.0.0.1:8991/mqtt/superuser", "https://[::1]:8991/mqtt/superuser" +#auth.http.super_req.endpoint = "http://127.0.0.1:8991/mqtt/superuser" ## Value: post | get #auth.http.super_req.method = post @@ -62,7 +62,7 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P ## - %k: websocket cookie ## ## Value: Params -#auth.http.super_req.params = clientid=%c,username=%u +#auth.http.super_req.params = "clientid=%c,username=%u" ##-------------------------------------------------------------------- ## ACL request. @@ -71,8 +71,8 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P ## ## Value: URL ## -## Examples: http://127.0.0.1:8991/mqtt/acl, https://[::1]:8991/mqtt/acl -auth.http.acl_req = http://127.0.0.1:8991/mqtt/acl +## Examples: "http://127.0.0.1:8991/mqtt/acl", "https://[::1]:8991/mqtt/acl" +auth.http.acl_req.endpoint = "http://127.0.0.1:8991/mqtt/acl" ## Value: post | get auth.http.acl_req.method = get @@ -92,7 +92,7 @@ auth.http.acl_req.content_type = x-www-form-urlencoded ## - %k: websocket cookie ## ## Value: Params -auth.http.acl_req.params = access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m +auth.http.acl_req.params = "access=%A,username=%u,clientid=%c,ipaddr=%a,topic=%t,mountpoint=%m" ##------------------------------------------------------------------------------ ## Http Reqeust options @@ -144,22 +144,22 @@ auth.http.request.retry_backoff = 2.0 ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## auth.http.ssl.cacertfile = {{ platform_etc_dir }}/certs/ca.pem +## auth.http.ssl.cacertfile = "{{ platform_etc_dir }}/certs/ca.pem" ## The path to a file containing the client's certificate. ## ## Value: File -## auth.http.ssl.certfile = {{ platform_etc_dir }}/certs/client-cert.pem +## auth.http.ssl.certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" ## Path to a file containing the client's private PEM-encoded key. ## ## Value: File -## auth.http.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +## auth.http.ssl.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ##-------------------------------------------------------------------- ## HTTP Request Headers ## -## Example: auth.http.header.Accept-Encoding = * +## Example: auth.http.header.Accept-Encoding = "*" ## ## Value: String -## auth.http.header.Accept = */* +## auth.http.header.Accept = "*/*" diff --git a/apps/emqx_auth_http/priv/emqx_auth_http.schema b/apps/emqx_auth_http/priv/emqx_auth_http.schema index 4f4289db0..95d7e5b5e 100644 --- a/apps/emqx_auth_http/priv/emqx_auth_http.schema +++ b/apps/emqx_auth_http/priv/emqx_auth_http.schema @@ -1,6 +1,6 @@ %%-*- mode: erlang -*- %% emqx_auth_http config mapping -{mapping, "auth.http.auth_req", "emqx_auth_http.auth_req", [ +{mapping, "auth.http.auth_req.endpoint", "emqx_auth_http.auth_req", [ {datatype, string} ]}. @@ -19,7 +19,7 @@ ]}. {translation, "emqx_auth_http.auth_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.auth_req", Conf) of + case cuttlefish:conf_get("auth.http.auth_req.endpoint", Conf) of undefined -> cuttlefish:unset(); Url -> Params = cuttlefish:conf_get("auth.http.auth_req.params", Conf), @@ -30,7 +30,7 @@ end end}. -{mapping, "auth.http.super_req", "emqx_auth_http.super_req", [ +{mapping, "auth.http.super_req.endpoint", "emqx_auth_http.super_req", [ {datatype, string} ]}. @@ -49,7 +49,7 @@ end}. ]}. {translation, "emqx_auth_http.super_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.super_req", Conf, undefined) of + case cuttlefish:conf_get("auth.http.super_req.endpoint", Conf, undefined) of undefined -> cuttlefish:unset(); Url -> Params = cuttlefish:conf_get("auth.http.super_req.params", Conf), [{url, Url}, {method, cuttlefish:conf_get("auth.http.super_req.method", Conf)}, @@ -58,7 +58,7 @@ end}. end end}. -{mapping, "auth.http.acl_req", "emqx_auth_http.acl_req", [ +{mapping, "auth.http.acl_req.endpoint", "emqx_auth_http.acl_req", [ {default, undefined}, {datatype, string} ]}. @@ -78,7 +78,7 @@ end}. ]}. {translation, "emqx_auth_http.acl_req", fun(Conf) -> - case cuttlefish:conf_get("auth.http.acl_req", Conf, undefined) of + case cuttlefish:conf_get("auth.http.acl_req.endpoint", Conf, undefined) of undefined -> cuttlefish:unset(); Url -> Params = cuttlefish:conf_get("auth.http.acl_req.params", Conf), [{url, Url}, @@ -166,4 +166,4 @@ end}. {Field, Value} end, cuttlefish_variable:filter_by_prefix("auth.http.header", Conf)) -end}. \ No newline at end of file +end}. diff --git a/apps/emqx_auth_http/rebar.config b/apps/emqx_auth_http/rebar.config index 0f2e48298..a7b8dd778 100644 --- a/apps/emqx_auth_http/rebar.config +++ b/apps/emqx_auth_http/rebar.config @@ -22,7 +22,7 @@ {profiles, [{test, [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + [ {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "v1.2.2"}}} ]} ]} diff --git a/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf b/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf index 5a599ca23..2aeb99d74 100644 --- a/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf +++ b/apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf @@ -10,13 +10,13 @@ auth.jwt.secret = emqxsecret ## RSA or ECDSA public key file. ## ## Value: File -#auth.jwt.pubkey = etc/certs/jwt_public_key.pem +#auth.jwt.pubkey = "etc/certs/jwt_public_key.pem" ## The JWKs server address ## ## see: http://self-issued.info/docs/draft-ietf-jose-json-web-key.html ## -#auth.jwt.jwks = https://127.0.0.1:8080/jwks +#auth.jwt.jwks.endpoint = "https://127.0.0.1:8080/jwks" ## The JWKs refresh interval ## @@ -32,7 +32,7 @@ auth.jwt.from = password ## Enable to verify claims fields ## ## Value: on | off -auth.jwt.verify_claims = off +auth.jwt.verify_claims.enable = off ## The checklist of claims to validate ## @@ -42,4 +42,4 @@ auth.jwt.verify_claims = off ## Variables: ## - %u: username ## - %c: clientid -#auth.jwt.verify_claims.username = %u +#auth.jwt.verify_claims.username = "%u" diff --git a/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema b/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema index 3d8de3678..10b2daa5e 100644 --- a/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema +++ b/apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema @@ -4,7 +4,7 @@ {datatype, string} ]}. -{mapping, "auth.jwt.jwks", "emqx_auth_jwt.jwks", [ +{mapping, "auth.jwt.jwks.endpoint", "emqx_auth_jwt.jwks", [ {datatype, string} ]}. @@ -26,7 +26,7 @@ {datatype, {enum, [raw, der]}} ]}. -{mapping, "auth.jwt.verify_claims", "emqx_auth_jwt.verify_claims", [ +{mapping, "auth.jwt.verify_claims.enable", "emqx_auth_jwt.verify_claims", [ {default, off}, {datatype, flag} ]}. @@ -36,7 +36,7 @@ ]}. {translation, "emqx_auth_jwt.verify_claims", fun(Conf) -> - case cuttlefish:conf_get("auth.jwt.verify_claims", Conf) of + case cuttlefish:conf_get("auth.jwt.verify_claims.enable", Conf) of false -> cuttlefish:unset(); true -> lists:foldr( diff --git a/apps/emqx_auth_jwt/rebar.config b/apps/emqx_auth_jwt/rebar.config index 4164d1fed..05b815239 100644 --- a/apps/emqx_auth_jwt/rebar.config +++ b/apps/emqx_auth_jwt/rebar.config @@ -20,6 +20,6 @@ {profiles, [{test, - [{deps, [{emqx_ct_helpers, {git, "http://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + [{deps, []} ]} ]}. diff --git a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf index 746510fb3..c849a7eec 100644 --- a/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf +++ b/apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf @@ -5,7 +5,7 @@ ## LDAP server list, seperated by ','. ## ## Value: String -auth.ldap.servers = 127.0.0.1 +auth.ldap.servers = "127.0.0.1" ## LDAP server port. ## @@ -20,7 +20,7 @@ auth.ldap.pool = 8 ## LDAP Bind DN. ## ## Value: DN -auth.ldap.bind_dn = cn=root,dc=emqx,dc=io +auth.ldap.bind_dn = "cn=root,dc=emqx,dc=io" ## LDAP Bind Password. ## @@ -37,7 +37,7 @@ auth.ldap.timeout = 30s ## Variables: ## ## Value: DN -auth.ldap.device_dn = ou=device,dc=emqx,dc=io +auth.ldap.device_dn = "ou=device,dc=emqx,dc=io" ## Specified ObjectClass ## @@ -63,15 +63,15 @@ auth.ldap.password.attributetype = userPassword ## Whether to enable SSL. ## ## Value: true | false -auth.ldap.ssl = false +auth.ldap.ssl.enable = false -#auth.ldap.ssl.certfile = etc/certs/cert.pem +#auth.ldap.ssl.certfile = "etc/certs/cert.pem" -#auth.ldap.ssl.keyfile = etc/certs/key.pem +#auth.ldap.ssl.keyfile = "etc/certs/key.pem" -#auth.ldap.ssl.cacertfile = etc/certs/cacert.pem +#auth.ldap.ssl.cacertfile = "etc/certs/cacert.pem" -#auth.ldap.ssl.verify = verify_peer +#auth.ldap.ssl.verify = "verify_peer" #auth.ldap.ssl.fail_if_no_peer_cert = true diff --git a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema index 554752a0b..a9b908fab 100644 --- a/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema +++ b/apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema @@ -31,7 +31,7 @@ {datatype, {duration, ms}} ]}. -{mapping, "auth.ldap.ssl", "emqx_auth_ldap.ldap", [ +{mapping, "auth.ldap.ssl.enable", "emqx_auth_ldap.ldap", [ {default, false}, {datatype, {enum, [true, false]}} ]}. @@ -85,7 +85,7 @@ {bind_password, BindPassword}, {pool, Pool}, {auto_reconnect, 2}], - case cuttlefish:conf_get("auth.ldap.ssl", Conf) of + case cuttlefish:conf_get("auth.ldap.ssl.enable", Conf) of true -> [{ssl, true}, {sslopts, Filter(SslOpts())}|Opts]; false -> [{ssl, false}|Opts] end diff --git a/apps/emqx_auth_ldap/rebar.config b/apps/emqx_auth_ldap/rebar.config index 48eaf812f..811468a7b 100644 --- a/apps/emqx_auth_ldap/rebar.config +++ b/apps/emqx_auth_ldap/rebar.config @@ -4,7 +4,7 @@ {profiles, [{test, - [{deps, [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + [{deps, []} ]} ]}. diff --git a/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf b/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf index ff74656cb..758df1a9c 100644 --- a/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf +++ b/apps/emqx_auth_mnesia/etc/emqx_auth_mnesia.conf @@ -10,12 +10,12 @@ auth.mnesia.password_hash = sha256 ## Examples ##auth.client.1.clientid = id ##auth.client.1.password = passwd -##auth.client.2.clientid = dev:devid +##auth.client.2.clientid = "dev:devid" ##auth.client.2.password = passwd2 -##auth.client.3.clientid = app:appid +##auth.client.3.clientid = "app:appid" ##auth.client.3.password = passwd3 -##auth.client.4.clientid = client~!@#$%^&*()_+ -##auth.client.4.password = passwd~!@#$%^&*()_+ +##auth.client.4.clientid = "client~!@#$%^&*()_+" +##auth.client.4.password = "passwd~!@#$%^&*()_+" ##-------------------------------------------------------------------- ## Username Authentication @@ -26,5 +26,5 @@ auth.mnesia.password_hash = sha256 ##auth.user.1.password = public ##auth.user.2.username = feng@emqtt.io ##auth.user.2.password = public -##auth.user.3.username = name~!@#$%^&*()_+ -##auth.user.3.password = pwsswd~!@#$%^&*()_+ +##auth.user.3.username = "name~!@#$%^&*()_+" +##auth.user.3.password = "pwsswd~!@#$%^&*()_+" diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index d2645c4e1..d05f088dd 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -17,7 +17,7 @@ auth.mongo.type = single ## Value: String ## ## Examples: 127.0.0.1:27017,127.0.0.2:27017... -auth.mongo.server = 127.0.0.1:27017 +auth.mongo.server = "127.0.0.1:27017" ## MongoDB pool size ## @@ -102,17 +102,17 @@ auth.mongo.topology.max_overflow = 0 auth.mongo.auth_query.password_hash = sha256 ## sha256 with salt suffix -## auth.mongo.auth_query.password_hash = sha256,salt +## auth.mongo.auth_query.password_hash = "sha256,salt" ## sha256 with salt prefix -## auth.mongo.auth_query.password_hash = salt,sha256 +## auth.mongo.auth_query.password_hash = "salt,sha256" ## bcrypt with salt prefix -## auth.mongo.auth_query.password_hash = salt,bcrypt +## auth.mongo.auth_query.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.mongo.auth_query.password_hash = pbkdf2,sha256,1000,20 +## auth.mongo.auth_query.password_hash = "pbkdf2,sha256,1000,20" ## Authentication query. auth.mongo.auth_query.collection = mqtt_user @@ -131,15 +131,15 @@ auth.mongo.auth_query.password_field = password ## - %d: subject of client TLS cert ## ## auth.mongo.auth_query.selector = {Field}={Placeholder} -auth.mongo.auth_query.selector = username=%u +auth.mongo.auth_query.selector = "username=%u" ## ------------------------------------------------- ## Super User Query ## ------------------------------------------------- auth.mongo.super_query.collection = mqtt_user auth.mongo.super_query.super_field = is_superuser -#auth.mongo.super_query.selector = username=%u, clientid=%c -auth.mongo.super_query.selector = username=%u +#auth.mongo.super_query.selector.1 = username=%u, clientid=%c +auth.mongo.super_query.selector = "username=%u" ## ACL Selector. ## @@ -150,8 +150,8 @@ auth.mongo.super_query.selector = username=%u ## ## With following 2 selectors configured: ## -## auth.mongo.acl_query.selector.1 = username=%u -## auth.mongo.acl_query.selector.2 = username=$all +## auth.mongo.acl_query.selector.1 = "username=%u" +## auth.mongo.acl_query.selector.2 = "username=$all" ## ## And if a client connected using username 'ilyas', ## then the following mongo command will be used to @@ -165,8 +165,8 @@ auth.mongo.super_query.selector = username=%u ## ## Examples: ## -## auth.mongo.acl_query.selector.1 = username=%u,clientid=%c -## auth.mongo.acl_query.selector.2 = username=$all -## auth.mongo.acl_query.selector.3 = clientid=$all +## auth.mongo.acl_query.selector.1 = "username=%u,clientid=%c" +## auth.mongo.acl_query.selector.2 = "username=$all" +## auth.mongo.acl_query.selector.3 = "clientid=$all" auth.mongo.acl_query.collection = mqtt_acl -auth.mongo.acl_query.selector = username=%u +auth.mongo.acl_query.selector = "username=%u" diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index ebc5480ae..e00f66f59 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -41,7 +41,7 @@ {datatype, string} ]}. -{mapping, "auth.mongo.ssl", "emqx_auth_mongo.server", [ +{mapping, "auth.mongo.ssl.enable", "emqx_auth_mongo.server", [ {default, off}, {datatype, flag} ]}. @@ -99,7 +99,7 @@ true -> []; false -> [{r_mode, R}] end, - Ssl = case cuttlefish:conf_get("auth.mongo.ssl", Conf) of + Ssl = case cuttlefish:conf_get("auth.mongo.ssl.enable", Conf) of true -> Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index c502e36b1..8117a1d02 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -7,7 +7,7 @@ ## Value: Port | IP:Port ## ## Examples: 3306, 127.0.0.1:3306, localhost:3306 -auth.mysql.server = 127.0.0.1:3306 +auth.mysql.server = "127.0.0.1:3306" ## MySQL pool size. ## @@ -50,7 +50,7 @@ auth.mysql.database = mqtt ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.mysql.auth_query = select password from mqtt_user where username = '%u' limit 1 +auth.mysql.auth_query = "select password from mqtt_user where username = '%u' limit 1" ## auth.mysql.auth_query = select password_hash as password from mqtt_user where username = '%u' limit 1 ## Password hash. @@ -59,17 +59,17 @@ auth.mysql.auth_query = select password from mqtt_user where username = '%u' lim auth.mysql.password_hash = sha256 ## sha256 with salt prefix -## auth.mysql.password_hash = salt,sha256 +## auth.mysql.password_hash = "salt,sha256" ## bcrypt with salt only prefix -## auth.mysql.password_hash = salt,bcrypt +## auth.mysql.password_hash = "salt,bcrypt" ## sha256 with salt suffix -## auth.mysql.password_hash = sha256,salt +## auth.mysql.password_hash = "sha256,salt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.mysql.password_hash = pbkdf2,sha256,1000,20 +## auth.mysql.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query. ## @@ -81,7 +81,7 @@ auth.mysql.password_hash = sha256 ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.mysql.super_query = select is_superuser from mqtt_user where username = '%u' limit 1 +auth.mysql.super_query = "select is_superuser from mqtt_user where username = '%u' limit 1" ## ACL query. ## @@ -93,12 +93,12 @@ auth.mysql.super_query = select is_superuser from mqtt_user where username = '%u ## - %c: clientid ## ## Note: You can add the 'ORDER BY' statement to control the rules match order -auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c' +auth.mysql.acl_query = "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" ## Mysql ssl configuration. ## ## Value: on | off -## auth.mysql.ssl = off +## auth.mysql.ssl.enable = off ## CA certificate. ## diff --git a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema index 37eed8a5f..f5f01edc9 100644 --- a/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema +++ b/apps/emqx_auth_mysql/priv/emqx_auth_mysql.schema @@ -30,7 +30,7 @@ {datatype, string} ]}. -{mapping, "auth.mysql.ssl", "emqx_auth_mysql.server", [ +{mapping, "auth.mysql.ssl.enable", "emqx_auth_mysql.server", [ {default, off}, {datatype, flag} ]}. @@ -82,7 +82,7 @@ {query_timeout, Timeout}, {keep_alive, true}], Options1 = - case cuttlefish:conf_get("auth.mysql.ssl", Conf) of + case cuttlefish:conf_get("auth.mysql.ssl.enable", Conf) of true -> CA = cuttlefish:conf_get("auth.mysql.ssl.cafile", Conf), Cert = cuttlefish:conf_get("auth.mysql.ssl.certfile", Conf), diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index c3c6e2800..616499849 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -6,8 +6,8 @@ ## ## Value: Port | IP:Port ## -## Examples: 5432, 127.0.0.1:5432, localhost:5432 -auth.pgsql.server = 127.0.0.1:5432 +## Examples: 5432, "127.0.0.1:5432", "localhost:5432" +auth.pgsql.server = "127.0.0.1:5432" ## PostgreSQL pool size. ## @@ -37,7 +37,7 @@ auth.pgsql.encoding = utf8 ## Whether to enable SSL connection. ## ## Value: on | off -auth.pgsql.ssl = off +auth.pgsql.ssl.enable = off ## SSL keyfile. ## @@ -64,7 +64,7 @@ auth.pgsql.ssl = off ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.pgsql.auth_query = select password from mqtt_user where username = '%u' limit 1 +auth.pgsql.auth_query = "select password from mqtt_user where username = '%u' limit 1" ## Password hash. ## @@ -72,17 +72,17 @@ auth.pgsql.auth_query = select password from mqtt_user where username = '%u' lim auth.pgsql.password_hash = sha256 ## sha256 with salt prefix -## auth.pgsql.password_hash = salt,sha256 +## auth.pgsql.password_hash = "salt,sha256" ## sha256 with salt suffix -## auth.pgsql.password_hash = sha256,salt +## auth.pgsql.password_hash = "sha256,salt" ## bcrypt with salt prefix -## auth.pgsql.password_hash = salt,bcrypt +## auth.pgsql.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.pgsql.password_hash = pbkdf2,sha256,1000,20 +## auth.pgsql.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query. ## @@ -94,7 +94,7 @@ auth.pgsql.password_hash = sha256 ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert ## -auth.pgsql.super_query = select is_superuser from mqtt_user where username = '%u' limit 1 +auth.pgsql.super_query = "select is_superuser from mqtt_user where username = '%u' limit 1" ## ACL query. Comment this query, the ACL will be disabled. ## @@ -106,5 +106,5 @@ auth.pgsql.super_query = select is_superuser from mqtt_user where username = '%u ## - %c: clientid ## ## Note: You can add the 'ORDER BY' statement to control the rules match order -auth.pgsql.acl_query = select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c' +auth.pgsql.acl_query = "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index 078158c0a..dd1d83cb0 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -30,7 +30,7 @@ {datatype, atom} ]}. -{mapping, "auth.pgsql.ssl", "emqx_auth_pgsql.server", [ +{mapping, "auth.pgsql.ssl.enable", "emqx_auth_pgsql.server", [ {default, off}, {datatype, flag} ]}. @@ -61,7 +61,7 @@ Passwd = cuttlefish:conf_get("auth.pgsql.password", Conf, ""), DB = cuttlefish:conf_get("auth.pgsql.database", Conf), Encoding = cuttlefish:conf_get("auth.pgsql.encoding", Conf), - Ssl = cuttlefish:conf_get("auth.pgsql.ssl", Conf), + Ssl = cuttlefish:conf_get("auth.pgsql.ssl.enable", Conf), Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, SslOpts = fun(Prefix) -> @@ -69,7 +69,7 @@ {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}]) end, - + TempHost = case inet:parse_address(PgHost) of {ok, IpAddr} -> IpAddr; diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 644b90e4e..2ca7cedc9 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -12,9 +12,9 @@ auth.redis.type = single ## Value: Port | IP:Port ## ## Single Redis Server: 127.0.0.1:6379, localhost:6379 -## Redis Sentinel: 127.0.0.1:26379,127.0.0.2:26379,127.0.0.3:26379 -## Redis Cluster: 127.0.0.1:6379,127.0.0.2:6379,127.0.0.3:6379 -auth.redis.server = 127.0.0.1:6379 +## Redis Sentinel: "127.0.0.1:26379,127.0.0.2:26379,127.0.0.3:26379" +## Redis Cluster: "127.0.0.1:6379,127.0.0.2:6379,127.0.0.3:6379" +auth.redis.server = "127.0.0.1:6379" ## Redis sentinel cluster name. ## @@ -52,10 +52,10 @@ auth.redis.database = 0 ## - %d: subject of client TLS cert ## ## Examples: -## - HGET mqtt_user:%u password -## - HMGET mqtt_user:%u password -## - HMGET mqtt_user:%u password salt -auth.redis.auth_cmd = HMGET mqtt_user:%u password +## - "HGET mqtt_user:%u password" +## - "HMGET mqtt_user:%u password" +## - "HMGET mqtt_user:%u password salt" +auth.redis.auth_cmd = "HMGET mqtt_user:%u password" ## Password hash. ## @@ -63,17 +63,17 @@ auth.redis.auth_cmd = HMGET mqtt_user:%u password auth.redis.password_hash = plain ## sha256 with salt prefix -## auth.redis.password_hash = salt,sha256 +## auth.redis.password_hash = "salt,sha256" ## sha256 with salt suffix -## auth.redis.password_hash = sha256,salt +## auth.redis.password_hash = "sha256,salt" ## bcrypt with salt prefix -## auth.redis.password_hash = salt,bcrypt +## auth.redis.password_hash = "salt,bcrypt" ## pbkdf2 with macfun iterations dklen ## macfun: md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512 -## auth.redis.password_hash = pbkdf2,sha256,1000,20 +## auth.redis.password_hash = "pbkdf2,sha256,1000,20" ## Superuser query command. ## @@ -84,7 +84,7 @@ auth.redis.password_hash = plain ## - %c: clientid ## - %C: common name of client TLS cert ## - %d: subject of client TLS cert -auth.redis.super_cmd = HGET mqtt_user:%u is_superuser +auth.redis.super_cmd = "HGET mqtt_user:%u is_superuser" ## ACL query command. ## @@ -93,12 +93,12 @@ auth.redis.super_cmd = HGET mqtt_user:%u is_superuser ## Variables: ## - %u: username ## - %c: clientid -auth.redis.acl_cmd = HGETALL mqtt_acl:%u +auth.redis.acl_cmd = "HGETALL mqtt_acl:%u" ## Redis ssl configuration. ## ## Value: on | off -#auth.redis.ssl = off +#auth.redis.ssl.enable = off ## CA certificate. ## diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index a70be6a8d..bfd572915 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -33,7 +33,7 @@ hidden ]}. -{mapping, "auth.redis.ssl", "emqx_auth_redis.options", [ +{mapping, "auth.redis.ssl.enable", "emqx_auth_redis.options", [ {default, off}, {datatype, flag} ]}. @@ -54,7 +54,7 @@ ]}. {translation, "emqx_auth_redis.options", fun(Conf) -> - Ssl = cuttlefish:conf_get("auth.redis.ssl", Conf, false), + Ssl = cuttlefish:conf_get("auth.redis.ssl.enable", Conf, false), case Ssl of true -> CA = cuttlefish:conf_get("auth.redis.ssl.cafile", Conf), diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index 023f986fc..5f4a8b8da 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -9,8 +9,8 @@ ## Bridge address: node name for local bridge, host:port for remote. ## ## Value: String -## Example: emqx@127.0.0.1, 127.0.0.1:1883 -bridge.mqtt.aws.address = 127.0.0.1:1883 +## Example: emqx@127.0.0.1, "127.0.0.1:1883" +bridge.mqtt.aws.address = "127.0.0.1:1883" ## Protocol version of the bridge. ## @@ -65,18 +65,18 @@ bridge.mqtt.aws.password = passwd ## Topics that need to be forward to AWS IoTHUB ## ## Value: String -## Example: topic1/#,topic2/# -bridge.mqtt.aws.forwards = topic1/#,topic2/# +## Example: "topic1/#,topic2/#" +bridge.mqtt.aws.forwards = "topic1/#,topic2/#" ## Forward messages to the mountpoint of an AWS IoTHUB ## ## Value: String -bridge.mqtt.aws.forward_mountpoint = bridge/aws/${node}/ +bridge.mqtt.aws.forward_mountpoint = "bridge/aws/${node}/" ## Need to subscribe to AWS topics ## ## Value: String -## bridge.mqtt.aws.subscription.1.topic = cmd/topic1 +## bridge.mqtt.aws.subscription.1.topic = "cmd/topic1" ## Need to subscribe to AWS topics QoS. ## @@ -86,7 +86,7 @@ bridge.mqtt.aws.forward_mountpoint = bridge/aws/${node}/ ## A mountpoint that receives messages from AWS IoTHUB ## ## Value: String -## bridge.mqtt.aws.receive_mountpoint = receive/aws/ +## bridge.mqtt.aws.receive_mountpoint = "receive/aws/" ## Bribge to remote server via SSL. @@ -97,28 +97,28 @@ bridge.mqtt.aws.ssl = off ## PEM-encoded CA certificates of the bridge. ## ## Value: File -bridge.mqtt.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +bridge.mqtt.aws.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Client SSL Certfile of the bridge. ## ## Value: File -bridge.mqtt.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem +bridge.mqtt.aws.certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" ## Client SSL Keyfile of the bridge. ## ## Value: File -bridge.mqtt.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +bridge.mqtt.aws.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ## SSL Ciphers used by the bridge. ## ## Value: String -bridge.mqtt.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +bridge.mqtt.aws.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'bridge.${BridgeName}.ciphers' and 'bridge.${BridgeName}.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#bridge.mqtt.aws.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#bridge.mqtt.aws.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Ping interval of a down bridge. ## @@ -129,7 +129,7 @@ bridge.mqtt.aws.keepalive = 60s ## TLS versions used by the bridge. ## ## Value: String -bridge.mqtt.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +bridge.mqtt.aws.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## Bridge reconnect time. ## @@ -159,7 +159,7 @@ bridge.mqtt.aws.max_inflight = 32 ## replayq works in a mem-only manner. ## ## Value: String -bridge.mqtt.aws.queue.replayq_dir = {{ platform_data_dir }}/replayq/emqx_aws_bridge/ +bridge.mqtt.aws.queue.replayq_dir = "{{ platform_data_dir }}/replayq/emqx_aws_bridge/" ## Replayq segment size ## diff --git a/apps/emqx_coap/etc/emqx_coap.conf b/apps/emqx_coap/etc/emqx_coap.conf index 0590a348e..d6dfa1a6f 100644 --- a/apps/emqx_coap/etc/emqx_coap.conf +++ b/apps/emqx_coap/etc/emqx_coap.conf @@ -4,13 +4,13 @@ ## The IP and UDP port that CoAP bind with. ## -## Default: 0.0.0.0:5683 +## Default: "0.0.0.0:5683" ## ## Examples: -## coap.bind.udp.x = 0.0.0.0:5683 | :::5683 | 127.0.0.1:5683 | ::1:5683 +## coap.bind.udp.x = "0.0.0.0:5683" | ":::5683" | "127.0.0.1:5683" | "::1:5683" ## -coap.bind.udp.1 = 0.0.0.0:5683 -##coap.bind.udp.2 = 0.0.0.0:6683 +coap.bind.udp.1 = "0.0.0.0:5683" +##coap.bind.udp.2 = "0.0.0.0:6683" ## Whether to enable statistics for CoAP clients. ## @@ -23,13 +23,13 @@ coap.enable_stats = off ## The DTLS port that CoAP is listening on. ## -## Default: 0.0.0.0:5684 +## Default: "0.0.0.0:5684" ## ## Examples: -## coap.bind.dtls.x = 0.0.0.0:5684 | :::5684 | 127.0.0.1:5684 | ::1:5684 +## coap.bind.dtls.x = "0.0.0.0:5684" | ":::5684" | "127.0.0.1:5684" | "::1:5684" ## -coap.bind.dtls.1 = 0.0.0.0:5684 -##coap.bind.dtls.2 = 0.0.0.0:6684 +coap.bind.dtls.1 = "0.0.0.0:5684" +##coap.bind.dtls.2 = "0.0.0.0:6684" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -43,17 +43,17 @@ coap.bind.dtls.1 = 0.0.0.0:5684 ## Private key file for DTLS ## ## Value: File -coap.dtls.keyfile = {{ platform_etc_dir }}/certs/key.pem +coap.dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Server certificate for DTLS. ## ## Value: File -coap.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem +coap.dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## PEM-encoded CA certificates for DTLS ## ## Value: File -## coap.dtls.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## coap.dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Used together with {verify, verify_peer} by an SSL server. If set to true, ## the server fails if the client does not have a certificate to send, that is, @@ -79,4 +79,4 @@ coap.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -coap.dtls.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +coap.dtls.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 0b85b4f18..ad9d500ab 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -21,8 +21,6 @@ {profiles, [{test, [{deps, - [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, - {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}} - ]} + [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}]} ]} ]}. diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 7c2125b4c..3c566aa33 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -20,7 +20,7 @@ dashboard.default_user.password = public ## Value: Port ## ## Examples: 18083 -dashboard.listener.http = 18083 +dashboard.listener.http.port = 18083 ## The acceptor pool for external Dashboard HTTP listener. ## @@ -50,7 +50,7 @@ dashboard.listener.http.ipv6_v6only = false ## Value: Port ## ## Examples: 18084 -## dashboard.listener.https = 18084 +## dashboard.listener.https.port = 18084 ## The acceptor pool for external Dashboard HTTPS listener. ## @@ -75,22 +75,22 @@ dashboard.listener.http.ipv6_v6only = false ## Path to the file containing the user's private PEM-encoded key. ## ## Value: File -## dashboard.listener.https.keyfile = etc/certs/key.pem +## dashboard.listener.https.keyfile = "etc/certs/key.pem" ## Path to a file containing the user certificate. ## ## Value: File -## dashboard.listener.https.certfile = etc/certs/cert.pem +## dashboard.listener.https.certfile = "etc/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## Value: File -## dashboard.listener.https.cacertfile = etc/certs/cacert.pem +## dashboard.listener.https.cacertfile = "etc/certs/cacert.pem" ## See: 'listener.ssl..dhfile' in emq.conf ## ## Value: File -## dashboard.listener.https.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## dashboard.listener.https.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## See: 'listener.ssl..vefify' in emq.conf ## @@ -105,12 +105,12 @@ dashboard.listener.http.ipv6_v6only = false ## TLS versions only to protect from POODLE attack. ## ## Value: String, seperated by ',' -## dashboard.listener.https.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## dashboard.listener.https.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## See: 'listener.ssl..ciphers' in emq.conf ## ## Value: Ciphers -## dashboard.listener.https.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## dashboard.listener.https.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## See: 'listener.ssl..secure_renegotiate' in emq.conf ## diff --git a/apps/emqx_dashboard/priv/emqx_dashboard.schema b/apps/emqx_dashboard/priv/emqx_dashboard.schema index fcc8f3489..65dba68d1 100644 --- a/apps/emqx_dashboard/priv/emqx_dashboard.schema +++ b/apps/emqx_dashboard/priv/emqx_dashboard.schema @@ -9,7 +9,7 @@ {datatype, string} ]}. -{mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [ +{mapping, "dashboard.listener.http.port", "emqx_dashboard.listeners", [ {datatype, integer} ]}. @@ -37,7 +37,7 @@ {datatype, {enum, [true, false]}} ]}. -{mapping, "dashboard.listener.https", "emqx_dashboard.listeners", [ +{mapping, "dashboard.listener.https.port", "emqx_dashboard.listeners", [ {datatype, integer} ]}. @@ -138,7 +138,7 @@ lists:map( fun(Proto) -> Prefix = "dashboard.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of undefined -> []; Port -> [{Proto, Port, case Proto of diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index f6f5213f7..b2758e705 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -8,8 +8,8 @@ ## The gRPC server url ## ## exhook.server.$name.url = url() -exhook.server.default.url = http://127.0.0.1:9000 +exhook.server.default.url = "http://127.0.0.1:9000" -#exhook.server.default.ssl.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -#exhook.server.default.ssl.certfile = {{ platform_etc_dir }}/certs/cert.pem -#exhook.server.default.ssl.keyfile = {{ platform_etc_dir }}/certs/key.pem +#exhook.server.default.ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +#exhook.server.default.ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" +#exhook.server.default.ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index ebeaddeab..da52056c0 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -41,7 +41,6 @@ {profiles, [{test, [{deps, - [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}} - ]} + []} ]} ]}. diff --git a/apps/emqx_exproto/etc/emqx_exproto.conf b/apps/emqx_exproto/etc/emqx_exproto.conf index a64153791..8f45b418f 100644 --- a/apps/emqx_exproto/etc/emqx_exproto.conf +++ b/apps/emqx_exproto/etc/emqx_exproto.conf @@ -5,9 +5,9 @@ exproto.server.http.port = 9100 exproto.server.https.port = 9101 -exproto.server.https.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -exproto.server.https.certfile = {{ platform_etc_dir }}/certs/cert.pem -exproto.server.https.keyfile = {{ platform_etc_dir }}/certs/key.pem +exproto.server.https.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +exproto.server.https.certfile = "{{ platform_etc_dir }}/certs/cert.pem" +exproto.server.https.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ##-------------------------------------------------------------------- ## Listeners @@ -20,12 +20,12 @@ exproto.server.https.keyfile = {{ platform_etc_dir }}/certs/key.pem ## ## Value: ://: ## -## Examples: tcp://0.0.0.0:7993 | ssl://127.0.0.1:7994 -exproto.listener.protoname = tcp://0.0.0.0:7993 +## Examples: "tcp://0.0.0.0:7993" | "ssl://127.0.0.1:7994" +exproto.listener.protoname.endpoint = "tcp://0.0.0.0:7993" ## The ConnectionHandler server address ## -exproto.listener.protoname.connection_handler_url = http://127.0.0.1:9001 +exproto.listener.protoname.connection_handler_url = "http://127.0.0.1:9001" #exproto.listener.protoname.connection_handler_certfile = #exproto.listener.protoname.connection_handler_cacertfile = @@ -62,8 +62,8 @@ exproto.listener.protoname.idle_timeout = 30s ## ## Value: ACL Rule ## -## Example: allow 192.168.0.0/24 -exproto.listener.protoname.access.1 = allow all +## Example: "allow 192.168.0.0/24" +exproto.listener.protoname.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed ## behind HAProxy or Nginx. @@ -146,27 +146,27 @@ exproto.listener.protoname.reuseaddr = true ## See: http://erlang.org/doc/man/ssl.html ## ## Value: String, seperated by ',' -#exproto.listener.protoname.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +#exproto.listener.protoname.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## Path to the file containing the user's private PEM-encoded key. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -#exproto.listener.protoname.keyfile = {{ platform_etc_dir }}/certs/key.pem +#exproto.listener.protoname.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -#exproto.listener.protoname.certfile = {{ platform_etc_dir }}/certs/cert.pem +#exproto.listener.protoname.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -#exproto.listener.protoname.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +#exproto.listener.protoname.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit @@ -183,7 +183,7 @@ exproto.listener.protoname.reuseaddr = true ## openssl dhparam -out dh-params.pem 2048 ## ## Value: File -#exproto.listener.protoname.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +#exproto.listener.protoname.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -218,13 +218,13 @@ exproto.listener.protoname.reuseaddr = true ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -#exproto.listener.protoname.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +#exproto.listener.protoname.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#exproto.listener.protoname.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#exproto.listener.protoname.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## SSL parameter renegotiation is a feature that allows a client and a server ## to renegotiate the parameters of the SSL connection on the fly. diff --git a/apps/emqx_exproto/priv/emqx_exproto.schema b/apps/emqx_exproto/priv/emqx_exproto.schema index fb114dc77..4bd215847 100644 --- a/apps/emqx_exproto/priv/emqx_exproto.schema +++ b/apps/emqx_exproto/priv/emqx_exproto.schema @@ -44,7 +44,7 @@ end}. %%-------------------------------------------------------------------- %% Listeners -{mapping, "exproto.listener.$proto", "emqx_exproto.listeners", [ +{mapping, "exproto.listener.$proto.endpoint", "emqx_exproto.listeners", [ {datatype, string} ]}. @@ -340,7 +340,7 @@ end}. Listeners = fun(Proto) -> Prefix = string:join(["exproto","listener", Proto], "."), Opts = HandlerOpts(Prefix) ++ ConnOpts(Prefix) ++ LisOpts(Prefix), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".endpoint", Conf, undefined) of undefined -> []; ListenOn0 -> case ParseListenOn(ListenOn0) of @@ -359,6 +359,6 @@ end}. end end end, - lists:flatten([Listeners(Proto) || {[_, "listener", Proto], ListenOn} + lists:flatten([Listeners(Proto) || {[_, "listener", Proto, "endpoint"], ListenOn} <- cuttlefish_variable:filter_by_prefix("exproto.listener", Conf)]) end}. diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 9dd5a2090..bc1a0532d 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -44,7 +44,6 @@ {profiles, [{test, [{deps, - [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.0"}}} - ]} + []} ]} ]}. diff --git a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf index c9baf6feb..c0cca7981 100644 --- a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf +++ b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf @@ -21,39 +21,39 @@ lwm2m.lifetime_max = 86400s # Placeholders supported: # '%e': Endpoint Name # '%a': IP Address -lwm2m.mountpoint = lwm2m/%e/ +lwm2m.mountpoint = "lwm2m/%e/" # The topic subscribed by the lwm2m client after it is connected # Placeholders supported: # '%e': Endpoint Name # '%a': IP Address -lwm2m.topics.command = dn/# +lwm2m.topics.command = "dn/#" # The topic to which the lwm2m client's response is published -lwm2m.topics.response = up/resp +lwm2m.topics.response = "up/resp" # The topic to which the lwm2m client's notify message is published -lwm2m.topics.notify = up/notify +lwm2m.topics.notify = "up/notify" # The topic to which the lwm2m client's register message is published -lwm2m.topics.register = up/resp +lwm2m.topics.register = "up/resp" # The topic to which the lwm2m client's update message is published -lwm2m.topics.update = up/resp +lwm2m.topics.update = "up/resp" # Dir where the object definition files can be found -lwm2m.xml_dir = {{ platform_etc_dir }}/lwm2m_xml +lwm2m.xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" ##-------------------------------------------------------------------- ## UDP Listener options ## The IP and port of the LwM2M Gateway ## -## Default: 0.0.0.0:5683 +## Default: "0.0.0.0:5683" ## Examples: -## lwm2m.bind.udp.x = 0.0.0.0:5683 | :::5683 | 127.0.0.1:5683 | ::1:5683 -lwm2m.bind.udp.1 = 0.0.0.0:5683 -#lwm2m.bind.udp.2 = 0.0.0.0:6683 +## lwm2m.bind.udp.x = "0.0.0.0:5683" | ":::5683" | "127.0.0.1:5683" | "::1:5683" +lwm2m.bind.udp.1 = "0.0.0.0:5683" +#lwm2m.bind.udp.2 = "0.0.0.0:6683" ## Socket options, used for performance tuning ## @@ -70,13 +70,13 @@ lwm2m.opts.read_packets = 20 ## The DTLS port that LwM2M is listening on. ## -## Default: 0.0.0.0:5684 +## Default: "0.0.0.0:5684" ## ## Examples: -## lwm2m.bind.dtls.x = 0.0.0.0:5684 | :::5684 | 127.0.0.1:5684 | ::1:5684 +## lwm2m.bind.dtls.x = "0.0.0.0:5684" | ":::5684" | "127.0.0.1:5684" | "::1:5684" ## -lwm2m.bind.dtls.1 = 0.0.0.0:5684 -#lwm2m.bind.dtls.2 = 0.0.0.0:6684 +lwm2m.bind.dtls.1 = "0.0.0.0:5684" +#lwm2m.bind.dtls.2 = "0.0.0.0:6684" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -90,17 +90,17 @@ lwm2m.bind.dtls.1 = 0.0.0.0:5684 ## Private key file for DTLS ## ## Value: File -lwm2m.dtls.keyfile = {{ platform_etc_dir }}/certs/key.pem +lwm2m.dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Server certificate for DTLS. ## ## Value: File -lwm2m.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem +lwm2m.dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## PEM-encoded CA certificates for DTLS ## ## Value: File -#lwm2m.dtls.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +#lwm2m.dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Used together with {verify, verify_peer} by an SSL server. If set to true, ## the server fails if the client does not have a certificate to send, that is, @@ -126,11 +126,11 @@ lwm2m.dtls.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -lwm2m.dtls.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +lwm2m.dtls.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## ## Note that 'lwm2m.dtls.ciphers' and 'lwm2m.dtls.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#lwm2m.dtls.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#lwm2m.dtls.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index b9dea9bb8..fbfcdc02f 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -5,7 +5,6 @@ {profiles, [{test, [{deps, [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, - {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} ]} ]} diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index 31a3c1dc5..a01a6aaf1 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -23,7 +23,7 @@ management.default_application.secret = public ##-------------------------------------------------------------------- ## HTTP Listener -management.listener.http = 8081 +management.listener.http.port = 8081 management.listener.http.acceptors = 2 management.listener.http.max_clients = 512 management.listener.http.backlog = 512 @@ -35,18 +35,18 @@ management.listener.http.ipv6_v6only = false ##-------------------------------------------------------------------- ## HTTPS Listener -## management.listener.https = 8081 +## management.listener.https.port = 8081 ## management.listener.https.acceptors = 2 ## management.listener.https.max_clients = 512 ## management.listener.https.backlog = 512 ## management.listener.https.send_timeout = 15s ## management.listener.https.send_timeout_close = on -## management.listener.https.certfile = etc/certs/cert.pem -## management.listener.https.keyfile = etc/certs/key.pem -## management.listener.https.cacertfile = etc/certs/cacert.pem +## management.listener.https.certfile = "etc/certs/cert.pem" +## management.listener.https.keyfile = "etc/certs/key.pem" +## management.listener.https.cacertfile = "etc/certs/cacert.pem" ## management.listener.https.verify = verify_peer -## management.listener.https.tls_versions = tlsv1.2,tlsv1.1,tlsv1 -## management.listener.https.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## management.listener.https.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" +## management.listener.https.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## management.listener.https.fail_if_no_peer_cert = true ## management.listener.https.inet6 = false ## management.listener.https.ipv6_v6only = false diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema index 343a70de6..4e887809e 100644 --- a/apps/emqx_management/priv/emqx_management.schema +++ b/apps/emqx_management/priv/emqx_management.schema @@ -21,7 +21,7 @@ {datatype, string} ]}. -{mapping, "management.listener.http", "emqx_management.listeners", [ +{mapping, "management.listener.http.port", "emqx_management.listeners", [ {datatype, [integer, ip]} ]}. @@ -85,7 +85,7 @@ {datatype, {enum, [true, false]}} ]}. -{mapping, "management.listener.https", "emqx_management.listeners", [ +{mapping, "management.listener.https.port", "emqx_management.listeners", [ {datatype, [integer, ip]} ]}. @@ -225,7 +225,7 @@ end}. lists:foldl( fun(Proto, Acc) -> Prefix = "management.listener." ++ atom_to_list(Proto), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of undefined -> Acc; Port -> [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix) diff --git a/apps/emqx_prometheus/etc/emqx_prometheus.conf b/apps/emqx_prometheus/etc/emqx_prometheus.conf index 7bfa22095..92e0d850d 100644 --- a/apps/emqx_prometheus/etc/emqx_prometheus.conf +++ b/apps/emqx_prometheus/etc/emqx_prometheus.conf @@ -5,7 +5,7 @@ ## The Prometheus Push Gateway URL address ## ## Note: You can comment out this line to disable it -prometheus.push.gateway.server = http://127.0.0.1:9091 +prometheus.push.gateway.server = "http://127.0.0.1:9091" ## The metrics data push interval (millisecond) ## diff --git a/apps/emqx_psk_file/etc/emqx_psk_file.conf b/apps/emqx_psk_file/etc/emqx_psk_file.conf index 3cee1c926..88c5bbdb1 100644 --- a/apps/emqx_psk_file/etc/emqx_psk_file.conf +++ b/apps/emqx_psk_file/etc/emqx_psk_file.conf @@ -1,2 +1,2 @@ -psk.file.path = {{ platform_etc_dir }}/psk.txt -psk.file.delimiter = : \ No newline at end of file +psk.file.path = "{{ platform_etc_dir }}/psk.txt" +psk.file.delimiter = ":" diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index 0a883cee5..4db438a98 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -37,5 +37,5 @@ retainer.max_payload_size = 1MB ## - 30m: 30 minutes ## - 20s: 20 seconds ## -## Defaut: 0 +## Default: 0 retainer.expiry_interval = 0 diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 9557780e8..7e762cb72 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -18,7 +18,7 @@ {profiles, [{test, [{deps, - [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + [ {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]} ]} ]}. diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index 2fe946779..556c59970 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -32,7 +32,7 @@ rule_engine.ignore_sys_message = on ## ## QoS-Level: qos0/qos1/qos2 -#rule_engine.events.client_connected = on, qos1 +#rule_engine.events.client_connected = "on, qos1" rule_engine.events.client_connected = off rule_engine.events.client_disconnected = off rule_engine.events.session_subscribed = off diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index a857ad60a..7b1341dff 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -2408,7 +2408,7 @@ start_apps() -> [start_apps(App, SchemaFile, ConfigFile) || {App, SchemaFile, ConfigFile} <- [{emqx, deps_path(emqx, "priv/emqx.schema"), - deps_path(emqx, "etc/emqx.conf")}, + deps_path(emqx, "etc/emqx.conf.rendered")}, {emqx_rule_engine, local_path("priv/emqx_rule_engine.schema"), local_path("etc/emqx_rule_engine.conf")}]]. @@ -2420,7 +2420,7 @@ start_apps(App, SchemaFile, ConfigFile) -> read_schema_configs(App, SchemaFile, ConfigFile) -> ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), Schema = cuttlefish_schema:files([SchemaFile]), - Conf = conf_parse:file(ConfigFile), + {ok, Conf} = hocon:load(ConfigFile, #{format => proplists}), NewConfig = cuttlefish_generator:map(Schema, Conf), Vals = proplists:get_value(App, NewConfig, []), [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. diff --git a/apps/emqx_sn/etc/emqx_sn.conf b/apps/emqx_sn/etc/emqx_sn.conf index 6572812c1..e05f1e7be 100644 --- a/apps/emqx_sn/etc/emqx_sn.conf +++ b/apps/emqx_sn/etc/emqx_sn.conf @@ -6,7 +6,7 @@ ## ## Value: IP:Port | Port ## -## Examples: 1884, 127.0.0.1:1884, ::1:1884 +## Examples: 1884, "127.0.0.1:1884", "::1:1884" mqtt.sn.port = 1884 ## The duration that emqx-sn broadcast ADVERTISE message through. @@ -37,8 +37,8 @@ mqtt.sn.idle_timeout = 30s ## The pre-defined topic name corresponding to the pre-defined topic id of N. ## Note that the pre-defined topic id of 0 is reserved. mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = /predefined/topic/name/hello -mqtt.sn.predefined.topic.2 = /predefined/topic/name/nice +mqtt.sn.predefined.topic.1 = "/predefined/topic/name/hello" +mqtt.sn.predefined.topic.2 = "/predefined/topic/name/nice" ## Default username for MQTT-SN. This parameter is optional. If specified, ## emq-sn will connect EMQ core with this username. It is useful if any auth diff --git a/apps/emqx_sn/priv/emqx_sn.schema b/apps/emqx_sn/priv/emqx_sn.schema index a585c1037..edc76db37 100644 --- a/apps/emqx_sn/priv/emqx_sn.schema +++ b/apps/emqx_sn/priv/emqx_sn.schema @@ -1,23 +1,19 @@ %%-*- mode: erlang -*- %% emqx_sn config mapping {mapping, "mqtt.sn.port", "emqx_sn.port", [ - {default, "1884"}, - {datatype, string} + {default, 1884}, + {datatype, [integer, ip]} ]}. {translation, "emqx_sn.port", fun(Conf) -> - case re:split(cuttlefish:conf_get("mqtt.sn.port", Conf, ""), ":", [{return, list}]) of - [Port] -> - {{0,0,0,0}, list_to_integer(Port)}; - Tokens -> - Port = lists:last(Tokens), - IP = case inet:parse_address(lists:flatten(lists:join(":", Tokens -- [Port]))) of - {error, Reason} -> - throw({invalid_ip_address, Reason}); - {ok, X} -> X - end, - Port1 = list_to_integer(Port), - {IP, Port1} + case cuttlefish:conf_get("mqtt.sn.port", Conf, undefined) of + Port when is_integer(Port) -> + {{0,0,0,0}, Port}; + {Ip, Port} -> + case inet:parse_address(Ip) of + {ok ,R} -> {R, Port}; + _ -> {Ip, Port} + end end end}. diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 5fecbb815..cbdac78f6 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -2,8 +2,7 @@ {plugins, [rebar3_proper]}. {deps, - [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}}, - {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + [{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/apps/emqx_stomp/etc/emqx_stomp.conf b/apps/emqx_stomp/etc/emqx_stomp.conf index e47f40b54..d8876670e 100644 --- a/apps/emqx_stomp/etc/emqx_stomp.conf +++ b/apps/emqx_stomp/etc/emqx_stomp.conf @@ -8,7 +8,7 @@ ## The Port that stomp listener will bind. ## ## Value: Port -stomp.listener = 61613 +stomp.listener.port = 61613 ## The acceptor pool for stomp listener. ## @@ -28,22 +28,22 @@ stomp.listener.max_connections = 512 ## Path to the file containing the user's private PEM-encoded key. ## ## Value: File -## stomp.listener.keyfile = etc/certs/key.pem +## stomp.listener.keyfile = "etc/certs/key.pem" ## Path to a file containing the user certificate. ## ## Value: File -## stomp.listener.certfile = etc/certs/cert.pem +## stomp.listener.certfile = "etc/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## Value: File -## stomp.listener.cacertfile = etc/certs/cacert.pem +## stomp.listener.cacertfile = "etc/certs/cacert.pem" ## See: 'listener.ssl..dhfile' in emq.conf ## ## Value: File -## stomp.listener.dhfile = etc/certs/dh-params.pem +## stomp.listener.dhfile = "etc/certs/dh-params.pem" ## See: 'listener.ssl..vefify' in emq.conf ## @@ -58,7 +58,7 @@ stomp.listener.max_connections = 512 ## TLS versions only to protect from POODLE attack. ## ## Value: String, seperated by ',' -## stomp.listener.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## stomp.listener.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## SSL Handshake timeout. ## @@ -68,7 +68,7 @@ stomp.listener.max_connections = 512 ## See: 'listener.ssl..ciphers' in emq.conf ## ## Value: Ciphers -## stomp.listener.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +## stomp.listener.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## See: 'listener.ssl..secure_renegotiate' in emq.conf ## diff --git a/apps/emqx_stomp/priv/emqx_stomp.schema b/apps/emqx_stomp/priv/emqx_stomp.schema index c77cc297e..32a3c272b 100644 --- a/apps/emqx_stomp/priv/emqx_stomp.schema +++ b/apps/emqx_stomp/priv/emqx_stomp.schema @@ -1,7 +1,7 @@ %%-*- mode: erlang -*- %% emqx_stomp config mapping -{mapping, "stomp.listener", "emqx_stomp.listener", [ +{mapping, "stomp.listener.port", "emqx_stomp.listener", [ {default, 61613}, {datatype, [integer, ip]} ]}. @@ -72,7 +72,7 @@ ]}. {translation, "emqx_stomp.listener", fun(Conf) -> - Port = cuttlefish:conf_get("stomp.listener", Conf), + Port = cuttlefish:conf_get("stomp.listener.port", Conf), Acceptors = cuttlefish:conf_get("stomp.listener.acceptors", Conf), MaxConnections = cuttlefish:conf_get("stomp.listener.max_connections", Conf), Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, diff --git a/apps/emqx_telemetry/etc/emqx_telemetry.conf b/apps/emqx_telemetry/etc/emqx_telemetry.conf index 041b54f60..326ef0508 100644 --- a/apps/emqx_telemetry/etc/emqx_telemetry.conf +++ b/apps/emqx_telemetry/etc/emqx_telemetry.conf @@ -13,8 +13,8 @@ telemetry.enabled = true ## ## Value: String ## -## Default: https://telemetry.emqx.io/api/telemetry -telemetry.url = https://telemetry.emqx.io/api/telemetry +## Default: "https://telemetry.emqx.io/api/telemetry" +telemetry.url = "https://telemetry.emqx.io/api/telemetry" ## Interval for reporting telemetry data ## @@ -25,4 +25,4 @@ telemetry.url = https://telemetry.emqx.io/api/telemetry ## -s: second ## ## Default: 7d -telemetry.report_interval = 7d \ No newline at end of file +telemetry.report_interval = 7d diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf index 41821fdc9..531d37c86 100644 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ b/apps/emqx_web_hook/etc/emqx_web_hook.conf @@ -5,16 +5,16 @@ ## Webhook URL ## ## Value: String -web.hook.url = http://127.0.0.1:8080 +web.hook.url = "http://127.0.0.1:8080" ## HTTP Headers -## +## ## Example: -## 1. web.hook.headers.content-type = application/json -## 2. web.hook.headers.accept = * +## 1. web.hook.headers.content-type = "application/json" +## 2. web.hook.headers.accept = "*" ## ## Value: String -web.hook.headers.content-type = application/json +web.hook.headers.content-type = "application/json" ## The encoding format of the payload field in the HTTP body ## The payload field only appears in the on_message_publish and on_message_delivered actions @@ -54,15 +54,15 @@ web.hook.pool_size = 32 ## ## Format: ## web.hook.rule.. = -#web.hook.rule.client.connect.1 = {"action": "on_client_connect"} -#web.hook.rule.client.connack.1 = {"action": "on_client_connack"} -#web.hook.rule.client.connected.1 = {"action": "on_client_connected"} -#web.hook.rule.client.disconnected.1 = {"action": "on_client_disconnected"} -#web.hook.rule.client.subscribe.1 = {"action": "on_client_subscribe"} -#web.hook.rule.client.unsubscribe.1 = {"action": "on_client_unsubscribe"} -#web.hook.rule.session.subscribed.1 = {"action": "on_session_subscribed"} -#web.hook.rule.session.unsubscribed.1 = {"action": "on_session_unsubscribed"} -#web.hook.rule.session.terminated.1 = {"action": "on_session_terminated"} -#web.hook.rule.message.publish.1 = {"action": "on_message_publish"} -#web.hook.rule.message.delivered.1 = {"action": "on_message_delivered"} -#web.hook.rule.message.acked.1 = {"action": "on_message_acked"} +#web.hook.rule.client.connect.1 = "{"action": "on_client_connect"}" +#web.hook.rule.client.connack.1 = "{"action": "on_client_connack"}" +#web.hook.rule.client.connected.1 = "{"action": "on_client_connected"}" +#web.hook.rule.client.disconnected.1 = "{"action": "on_client_disconnected"}" +#web.hook.rule.client.subscribe.1 = "{"action": "on_client_subscribe"}" +#web.hook.rule.client.unsubscribe.1 = "{"action": "on_client_unsubscribe"}" +#web.hook.rule.session.subscribed.1 = "{"action": "on_session_subscribed"}" +#web.hook.rule.session.unsubscribed.1 = "{"action": "on_session_unsubscribed"}" +#web.hook.rule.session.terminated.1 = "{"action": "on_session_terminated"}" +#web.hook.rule.message.publish.1 = "{"action": "on_message_publish"}" +#web.hook.rule.message.delivered.1 = "{"action": "on_message_delivered"}" +#web.hook.rule.message.acked.1 = ""{"action": "on_message_acked"}" diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config index a923a9906..c7a97cc62 100644 --- a/apps/emqx_web_hook/rebar.config +++ b/apps/emqx_web_hook/rebar.config @@ -24,8 +24,7 @@ [{test, [{erl_opts, [export_all, nowarn_export_all]}, {deps, - [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.0"}}}, - {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}, + [ {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}} ]} ]} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 2d82d4a5e..ab47864e1 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -3,7 +3,7 @@ ## Declare variables to be passed into your templates. ## It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. -replicaCount: 3 +replicaCount: 3 image: repository: emqx/emqx pullPolicy: IfNotPresent @@ -42,7 +42,7 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://github.com/emqx/emqx-docker#emq-x-configuration) emqxConfig: - EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__APISERVER: \"https://kubernetes.default.svc:443\" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note：Hostname is only supported after v4.0-rc.2 @@ -86,7 +86,7 @@ service: ## Port for MQTT ## mqtt: 1883 - ## Port for MQTT(SSL) + ## Port for MQTT(SSL) ## mqttssl: 8883 ## Port for mgmt API diff --git a/deploy/docker/start.sh b/deploy/docker/start.sh index e4e590363..c728d2a0c 100755 --- a/deploy/docker/start.sh +++ b/deploy/docker/start.sh @@ -38,7 +38,7 @@ tail -f /opt/emqx/log/erlang.log.1 & # and docker dispatching system can known and restart this container. IDLE_TIME=0 MGMT_CONF='/opt/emqx/etc/plugins/emqx_management.conf' -MGMT_PORT=$(sed -n -r '/^management.listener.http[ \t]=[ \t].*$/p' $MGMT_CONF | sed -r 's/^management.listener.http = (.*)$/\1/g') +MGMT_PORT=$(sed -n -r '/^management.listener.http.port[ \t]=[ \t].*$/p' $MGMT_CONF | sed -r 's/^management.listener.http.port = (.*)$/\1/g') while [ $IDLE_TIME -lt 5 ]; do IDLE_TIME=$(expr $IDLE_TIME + 1) if curl http://localhost:${MGMT_PORT}/status >/dev/null 2>&1; then diff --git a/etc/emqx.conf b/etc/emqx.conf index 0414d3c61..dbaf890ad 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -58,7 +58,7 @@ cluster.autoclean = 5m ## Node list of the cluster. ## ## Value: String -## cluster.static.seeds = emqx1@127.0.0.1,emqx2@127.0.0.1 +## cluster.static.seeds = "emqx1@127.0.0.1,emqx2@127.0.0.1" ##-------------------------------------------------------------------- ## Cluster using IP Multicast. @@ -66,19 +66,19 @@ cluster.autoclean = 5m ## IP Multicast Address. ## ## Value: IP Address -## cluster.mcast.addr = 239.192.0.1 +## cluster.mcast.addr = "239.192.0.1" ## Multicast Ports. ## ## Value: Port List -## cluster.mcast.ports = 4369,4370 +## cluster.mcast.ports = "4369,4370" ## Multicast Iface. ## ## Value: Iface Address ## -## Default: 0.0.0.0 -## cluster.mcast.iface = 0.0.0.0 +## Default: "0.0.0.0" +## cluster.mcast.iface = "0.0.0.0" ## Multicast Ttl. ## @@ -109,7 +109,7 @@ cluster.autoclean = 5m ## Etcd server list, seperated by ','. ## ## Value: String -## cluster.etcd.server = http://127.0.0.1:2379 +## cluster.etcd.server = "http://127.0.0.1:2379" ## The prefix helps build nodes path in etcd. Each node in the cluster ## will create a path in etcd: v2/keys/// @@ -127,18 +127,18 @@ cluster.autoclean = 5m ## Path to a file containing the client's private PEM-encoded key. ## ## Value: File -## cluster.etcd.ssl.keyfile = {{ platform_etc_dir }}/certs/client-key.pem +## cluster.etcd.ssl.keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" ## The path to a file containing the client's certificate. ## ## Value: File -## cluster.etcd.ssl.certfile = {{ platform_etc_dir }}/certs/client.pem +## cluster.etcd.ssl.certfile = "{{ platform_etc_dir }}/certs/client.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## cluster.etcd.ssl.cacertfile = {{ platform_etc_dir }}/certs/ca.pem +## cluster.etcd.ssl.cacertfile = "{{ platform_etc_dir }}/certs/ca.pem" ##-------------------------------------------------------------------- ## Cluster using Kubernetes @@ -146,7 +146,7 @@ cluster.autoclean = 5m ## Kubernetes API server list, seperated by ','. ## ## Value: String -## cluster.k8s.apiserver = http://10.110.111.204:8080 +## cluster.k8s.apiserver = "http://10.110.111.204:8080" ## The service name helps lookup EMQ nodes in the cluster. ## @@ -194,7 +194,7 @@ node.cookie = emqxsecretcookie ## Data dir for the node ## ## Value: Folder -node.data_dir = {{ platform_data_dir }} +node.data_dir = "{{ platform_data_dir }}" ## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable ## heartbeat, or set the value as 'on' @@ -271,14 +271,14 @@ node.global_gc_interval = 15m ## Crash dump log file. ## ## Value: Log file -node.crash_dump = {{ platform_log_dir }}/crash.dump +node.crash_dump = "{{ platform_log_dir }}/crash.dump" ## Specify SSL Options in the file if using SSL for Erlang Distribution. ## ## Value: File ## ## vm.args: -ssl_dist_optfile -## node.ssl_dist_optfile = {{ platform_etc_dir }}/ssl_dist.conf +## node.ssl_dist_optfile = "{{ platform_etc_dir }}/ssl_dist.conf" ## Sets the net_kernel tick time. TickTime is specified in seconds. ## Notice that all communicating nodes are to have the same TickTime @@ -427,7 +427,7 @@ log.level = warning ## The dir for log files. ## ## Value: Folder -log.dir = {{ platform_log_dir }} +log.dir = "{{ platform_log_dir }}" ## The log filename for logs of level specified in "log.level". ## @@ -450,7 +450,7 @@ log.file = emqx.log ## ## Value: on | off ## Default: on -log.rotation = on +log.rotation.enable = on ## Maximum size of each log file. ## @@ -569,7 +569,7 @@ log.rotation.count = 5 ## Value: MaxBurstCount,TimeWindow ## Default: disabled ## -#log.burst_limit = 20000, 1s +#log.burst_limit = "20000, 1s" ##-------------------------------------------------------------------- ## Authentication/Access Control @@ -589,7 +589,7 @@ acl_nomatch = allow ## Default ACL File. ## ## Value: File Name -acl_file = {{ platform_etc_dir }}/acl.conf +acl_file = "{{ platform_etc_dir }}/acl.conf" ## Whether to enable ACL cache. ## @@ -623,7 +623,7 @@ acl_deny_action = ignore ## 3. banned interval: the banned interval if a flapping is detected. ## ## Value: Integer,Duration,Duration -flapping_detect_policy = 30, 1m, 5m +flapping_detect_policy = "30, 1m, 5m" ##-------------------------------------------------------------------- ## MQTT Protocol @@ -681,7 +681,7 @@ mqtt.ignore_loop_deliver = false mqtt.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## mqtt.response_information = example @@ -722,7 +722,7 @@ zone.external.acl_deny_action = ignore ## messages | bytes passed through. ## ## Numbers delimited by `|'. Zero or negative is to disable. -zone.external.force_gc_policy = 16000|16MB +zone.external.force_gc_policy = "16000|16MB" ## Max message queue length and total heap size to force shutdown ## connection/session process. @@ -732,9 +732,9 @@ zone.external.force_gc_policy = 16000|16MB ## Numbers delimited by `|'. Zero or negative is to disable. ## ## Default: -## - 10000|64MB on ARCH_64 system -## - 1000|32MB on ARCH_32 sytem -#zone.external.force_shutdown_policy = 10000|64MB +## - "10000|64MB" on ARCH_64 system +## - "1000|32MB" on ARCH_32 sytem +#zone.external.force_shutdown_policy = "10000|64MB" ## Maximum MQTT packet size allowed. ## @@ -840,7 +840,7 @@ zone.external.max_mqueue_len = 1000 ## are treated equal ## ## Priority number [1-255] -## Example: topic/1=10,topic/2=8 +## Example: "topic/1=10,topic/2=8" ## NOTE: comma and equal signs are not allowed for priority topic names ## NOTE: messages for topics not in the priority table are treated as ## either highest or lowest priority depending on the configured @@ -867,29 +867,29 @@ zone.external.enable_flapping_detect = off ## ## Value: Number,Duration ## Example: 100 messages per 10 seconds. -#zone.external.rate_limit.conn_messages_in = 100,10s +#zone.external.rate_limit.conn_messages_in = "100,10s" ## Bytes limit for a external MQTT connections. ## ## Value: Number,Duration ## Example: 100KB incoming per 10 seconds. -#zone.external.rate_limit.conn_bytes_in = 100KB,10s +#zone.external.rate_limit.conn_bytes_in = "100KB,10s" ## Messages quota for the each of external MQTT connection. ## This value consumed by the number of recipient on a message. ## ## Value: Number, Duration ## -## Example: 100 messaegs per 1s -#zone.external.quota.conn_messages_routing = 100,1s +## Example: 100 messages per 1s +#zone.external.quota.conn_messages_routing = "100,1s" ## Messages quota for the all of external MQTT connections. ## This value consumed by the number of recipient on a message. ## ## Value: Number, Duration ## -## Example: 200000 messaegs per 1s -#zone.external.quota.overall_messages_routing = 200000,1s +## Example: 200000 messages per 1s +#zone.external.quota.overall_messages_routing = "200000,1s" ## All the topics will be prefixed with the mountpoint path if this option is enabled. ## @@ -898,7 +898,7 @@ zone.external.enable_flapping_detect = off ## - %u: username ## ## Value: String -## zone.external.mountpoint = devicebound/ +## zone.external.mountpoint = "devicebound/" ## Whether use username replace client id ## @@ -917,7 +917,7 @@ zone.external.ignore_loop_deliver = false zone.external.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## zone.external.response_information = example @@ -943,7 +943,7 @@ zone.internal.enable_acl = off zone.internal.acl_deny_action = ignore ## See zone.$name.force_gc_policy -## zone.internal.force_gc_policy = 128000|128MB +## zone.internal.force_gc_policy = "128000|128MB" ## See zone.$name.wildcard_subscription. ## @@ -988,8 +988,8 @@ zone.internal.enable_flapping_detect = off ## See zone.$name.force_shutdown_policy ## ## Default: -## - 10000|64MB on ARCH_64 system -## - 1000|32MB on ARCH_32 sytem +## - "10000|64MB" on ARCH_64 system +## - "1000|32MB" on ARCH_32 sytem #zone.internal.force_shutdown_policy = 10000|64MB ## All the topics will be prefixed with the mountpoint path if this option is enabled. @@ -999,7 +999,7 @@ zone.internal.enable_flapping_detect = off ## - %u: username ## ## Value: String -## zone.internal.mountpoint = cloudbound/ +## zone.internal.mountpoint = "cloudbound/" ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) ## @@ -1012,7 +1012,7 @@ zone.internal.ignore_loop_deliver = false zone.internal.strict_mode = false ## Specify the response information returned to the client -## +## ## Value: String ## zone.internal.response_information = example @@ -1033,8 +1033,8 @@ zone.internal.bypass_auth_plugins = true ## ## Value: IP:Port | Port ## -## Examples: 1883, 127.0.0.1:1883, ::1:1883 -listener.tcp.external = 0.0.0.0:1883 +## Examples: 1883, "127.0.0.1:1883", "::1:1883" +listener.tcp.external.endpoint = "0.0.0.0:1883" ## The acceptor pool for external MQTT/TCP listener. ## @@ -1069,8 +1069,8 @@ listener.tcp.external.zone = external ## ## Value: ACL Rule ## -## Example: allow 192.168.0.0/24 -listener.tcp.external.access.1 = allow all +## Example: "allow 192.168.0.0/24" +listener.tcp.external.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed ## behind HAProxy or Nginx. @@ -1165,8 +1165,8 @@ listener.tcp.external.reuseaddr = true ## ## Value: IP:Port, Port ## -## Examples: 11883, 127.0.0.1:11883, ::1:11883 -listener.tcp.internal = 127.0.0.1:11883 +## Examples: 11883, "127.0.0.1:11883", "::1:11883" +listener.tcp.internal.endpoint = "127.0.0.1:11883" ## The acceptor pool for internal MQTT/TCP listener. ## @@ -1262,8 +1262,8 @@ listener.tcp.internal.reuseaddr = true ## ## Value: IP:Port | Port ## -## Examples: 8883, 127.0.0.1:8883, ::1:8883 -listener.ssl.external = 8883 +## Examples: 8883, "127.0.0.1:8883", "::1:8883" +listener.ssl.external.endpoint = 8883 ## The acceptor pool for external MQTT/SSL listener. ## @@ -1295,7 +1295,7 @@ listener.ssl.external.zone = external ## See: listener.tcp.$name.access ## ## Value: ACL Rule -listener.ssl.external.access.1 = allow all +listener.ssl.external.access.1 = "allow all" ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. @@ -1317,7 +1317,7 @@ listener.ssl.external.access.1 = allow all ## See: http://erlang.org/doc/man/ssl.html ## ## Value: String, seperated by ',' -## listener.ssl.external.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## listener.ssl.external.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## TLS Handshake timeout. ## @@ -1341,20 +1341,20 @@ listener.ssl.external.handshake_timeout = 15s ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -listener.ssl.external.keyfile = {{ platform_etc_dir }}/certs/key.pem +listener.ssl.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: http://erlang.org/doc/man/ssl.html ## ## Value: File -listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem +listener.ssl.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. The CA certificates ## are used during server authentication and when building the client certificate chain. ## ## Value: File -## listener.ssl.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## listener.ssl.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## The Ephemeral Diffie-Helman key exchange is a very effective way of ## ensuring Forward Secrecy by exchanging a set of keys that never hit @@ -1371,7 +1371,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## openssl dhparam -out dh-params.pem 2048 ## ## Value: File -## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## A server only does x509-path validation in mode verify_peer, ## as it then sends a certificate request to the client (this @@ -1406,13 +1406,13 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -listener.ssl.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +listener.ssl.external.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -#listener.ssl.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +#listener.ssl.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## SSL parameter renegotiation is a feature that allows a client and a server ## to renegotiate the parameters of the SSL connection on the fly. @@ -1514,13 +1514,13 @@ listener.ssl.external.reuseaddr = true ## ## Value: IP:Port | Port ## -## Examples: 8083, 127.0.0.1:8083, ::1:8083 -listener.ws.external = 8083 +## Examples: 8083, "127.0.0.1:8083", "::1:8083" +listener.ws.external.endpoint = 8083 ## The path of WebSocket MQTT endpoint ## ## Value: URL Path -listener.ws.external.mqtt_path = /mqtt +listener.ws.external.mqtt_path = "/mqtt" ## The acceptor pool for external MQTT/WebSocket listener. ## @@ -1552,7 +1552,7 @@ listener.ws.external.zone = external ## See: listener.ws.$name.access ## ## Value: ACL Rule -listener.ws.external.access.1 = allow all +listener.ws.external.access.1 = "allow all" ## Verify if the protocol header is valid. Turn off for WeChat MiniApp. ## @@ -1712,13 +1712,13 @@ listener.ws.external.mqtt_piggyback = multiple ## ## Value: IP:Port | Port ## -## Examples: 8084, 127.0.0.1:8084, ::1:8084 -listener.wss.external = 8084 +## Examples: 8084, "127.0.0.1:8084", "::1:8084" +listener.wss.external.endpoint = 8084 ## The path of WebSocket MQTT endpoint ## ## Value: URL Path -listener.wss.external.mqtt_path = /mqtt +listener.wss.external.mqtt_path = "/mqtt" ## The acceptor pool for external MQTT/WebSocket/SSL listener. ## @@ -1752,7 +1752,7 @@ listener.wss.external.zone = external ## See: listener.tcp.$name.access. ## ## Value: ACL Rule -listener.wss.external.access.1 = allow all +listener.wss.external.access.1 = "allow all" ## See: listener.ws.external.verify_protocol_header ## @@ -1778,28 +1778,28 @@ listener.wss.external.verify_protocol_header = on ## See: listener.ssl.$name.tls_versions ## ## Value: String, seperated by ',' -## listener.wss.external.tls_versions = tlsv1.2,tlsv1.1,tlsv1 +## listener.wss.external.tls_versions = "tlsv1.2,tlsv1.1,tlsv1" ## Path to the file containing the user's private PEM-encoded key. ## ## See: listener.ssl.$name.keyfile ## ## Value: File -listener.wss.external.keyfile = {{ platform_etc_dir }}/certs/key.pem +listener.wss.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: listener.ssl.$name.certfile ## ## Value: File -listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem +listener.wss.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## ## See: listener.ssl.$name.cacert ## ## Value: File -## listener.wss.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## listener.wss.external.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" ## Maximum number of non-self-issued intermediate certificates that ## can follow the peer certificate in a valid certification path. @@ -1820,7 +1820,7 @@ listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## See: listener.ssl.$name.dhfile ## ## Value: File -## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## listener.ssl.external.dhfile = "{{ platform_etc_dir }}/certs/dh-params.pem" ## See: listener.ssl.$name.vefify ## @@ -1835,13 +1835,13 @@ listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## See: listener.ssl.$name.ciphers ## ## Value: Ciphers -listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +listener.wss.external.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.wss.external.ciphers' and 'listener.wss.external.psk_ciphers' cannot ## be configured at the same time. ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## listener.wss.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +## listener.wss.external.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## See: listener.ssl.$name.secure_renegotiate ## @@ -1991,7 +1991,7 @@ listener.wss.external.mqtt_piggyback = multiple ## The file to store loaded module names. ## ## Value: File -modules.loaded_file = {{ platform_data_dir }}/loaded_modules +modules.loaded_file = "{{ platform_data_dir }}/loaded_modules" ##-------------------------------------------------------------------- ## Presence Module @@ -2007,7 +2007,7 @@ module.presence.qos = 1 ## Subscribe the Topics automatically when client connected. ## ## Value: String -## module.subscription.1.topic = connected/%c/%u +## module.subscription.1.topic = "connected/%c/%u" ## Qos of the proxy subscription. ## @@ -2040,8 +2040,8 @@ module.presence.qos = 1 ## Rewrite Module ## {rewrite, Topic, Re, Dest} -## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1 -## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 +## module.rewrite.pub.rule.1 = "x/# ^x/y/(.+)$ z/y/$1" +## module.rewrite.sub.rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" ##------------------------------------------------------------------- ## Plugins @@ -2050,17 +2050,17 @@ module.presence.qos = 1 ## The etc dir for plugins' config. ## ## Value: Folder -plugins.etc_dir = {{ platform_etc_dir }}/plugins/ +plugins.etc_dir = "{{ platform_etc_dir }}/plugins/" ## The file to store loaded plugin names. ## ## Value: File -plugins.loaded_file = {{ platform_data_dir }}/loaded_plugins +plugins.loaded_file = "{{ platform_data_dir }}/loaded_plugins" ## The directory of extension plugins. ## ## Value: File -plugins.expand_plugins_dir = {{ platform_plugins_dir }}/ +plugins.expand_plugins_dir = "{{ platform_plugins_dir }}/" ##-------------------------------------------------------------------- ## Broker @@ -2148,7 +2148,6 @@ sysmon.long_gc = 0 ## Examples: ## - 2h: 2 hours ## - 30m: 30 minutes -## - 0.1s: 0.1 seconds ## - 100ms: 100 milliseconds ## ## Default: 0ms @@ -2240,12 +2239,12 @@ vm_mon.process_low_watermark = 60% ## - log ## - publish ## -## Default: log,publish -alarm.actions = log,publish +## Default: "log,publish" +alarm.actions = "log,publish" ## The maximum number of deactivated alarms ## -## Value: Integer +## Value: Integer ## ## Default: 1000 alarm.size_limit = 1000 diff --git a/priv/emqx.schema b/priv/emqx.schema index 8345f2f51..bc08e5cc6 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -482,7 +482,7 @@ end}. {datatype, integer} ]}. -{mapping, "log.rotation", "kernel.logger", [ +{mapping, "log.rotation.enable", "kernel.logger", [ {default, on}, {datatype, flag} ]}. @@ -576,7 +576,7 @@ end}. {translation, "kernel.logger", fun(Conf) -> LogTo = cuttlefish:conf_get("log.to", Conf), LogLevel = cuttlefish:conf_get("log.level", Conf), - LogType = case cuttlefish:conf_get("log.rotation", Conf) of + LogType = case cuttlefish:conf_get("log.rotation.enable", Conf) of true -> wrap; false -> halt end, @@ -1166,7 +1166,7 @@ end}. %%-------------------------------------------------------------------- %% TCP Listeners -{mapping, "listener.tcp.$name", "emqx.listeners", [ +{mapping, "listener.tcp.$name.endpoint", "emqx.listeners", [ {datatype, [integer, ip]} ]}. @@ -1267,7 +1267,7 @@ end}. %%-------------------------------------------------------------------- %% SSL Listeners -{mapping, "listener.ssl.$name", "emqx.listeners", [ +{mapping, "listener.ssl.$name.endpoint", "emqx.listeners", [ {datatype, [integer, ip]} ]}. @@ -1431,7 +1431,7 @@ end}. %%-------------------------------------------------------------------- %% MQTT/WebSocket Listeners -{mapping, "listener.ws.$name", "emqx.listeners", [ +{mapping, "listener.ws.$name.endpoint", "emqx.listeners", [ {datatype, [integer, ip]} ]}. @@ -1585,7 +1585,7 @@ end}. %%-------------------------------------------------------------------- %% MQTT/WebSocket/SSL Listeners -{mapping, "listener.wss.$name", "emqx.listeners", [ +{mapping, "listener.wss.$name.endpoint", "emqx.listeners", [ {datatype, [integer, ip]} ]}. @@ -1801,7 +1801,6 @@ end}. ]}. {translation, "emqx.listeners", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, @@ -1922,7 +1921,7 @@ end}. TcpListeners = fun(Type, Name) -> Prefix = string:join(["listener", Type, Name], "."), - ListenOnN = case cuttlefish:conf_get(Prefix, Conf, undefined) of + ListenOnN = case cuttlefish:conf_get(Prefix ++ ".endpoint", Conf, undefined) of undefined -> []; ListenOn -> case ListenOn of @@ -1939,7 +1938,7 @@ end}. end, SslListeners = fun(Type, Name) -> Prefix = string:join(["listener", Type, Name], "."), - case cuttlefish:conf_get(Prefix, Conf, undefined) of + case cuttlefish:conf_get(Prefix ++ ".endpoint", Conf, undefined) of undefined -> []; ListenOn -> @@ -1948,12 +1947,11 @@ end}. {ssl_options, SslOpts(Prefix)} | LisOpts(Prefix)]}] end end, - - lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} + lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name, "endpoint"], ListenOn} <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] ++ - [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} + [SslListeners(Type, Name) || {["listener", Type, Name, "endpoint"], ListenOn} <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) end}. diff --git a/rebar.config b/rebar.config index c1db4712c..8d5b47be0 100644 --- a/rebar.config +++ b/rebar.config @@ -47,7 +47,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.7.4"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.7.5"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {branch, "hocon"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.2.0"}}} diff --git a/rebar.config.erl b/rebar.config.erl index b203b1259..a5c5ae8c4 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -29,7 +29,7 @@ plugins() -> test_deps() -> [ {bbmustache, "1.10.0"} - , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.2"}}} + , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {branch, "hocon"}}} , meck ]. diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index ecfcf46ed..969ff6af5 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -67,14 +67,16 @@ mustache_vars() -> generate_config() -> Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), ConfFile = render_config_file(), - Conf = conf_parse:file(ConfFile), + {ok, Conf} = hocon:load(ConfFile, #{format => proplists}), cuttlefish_generator:map(Schema, Conf). set_app_env({App, Lists}) -> lists:foreach(fun({acl_file, _Var}) -> application:set_env(App, acl_file, local_path(["etc", "acl.conf"])); ({plugins_loaded_file, _Var}) -> - application:set_env(App, plugins_loaded_file, local_path(["test", "emqx_SUITE_data","loaded_plugins"])); + application:set_env(App, + plugins_loaded_file, + local_path(["test", "emqx_SUITE_data","loaded_plugins"])); ({Par, Var}) -> application:set_env(App, Par, Var) end, Lists). @@ -91,4 +93,4 @@ get_base_dir(Module) -> get_base_dir() -> get_base_dir(?MODULE). - + diff --git a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config index c690b88b1..0bc2e3d93 100644 --- a/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config +++ b/test/emqx_plugins_SUITE_data/emqx_mini_plugin/rebar.config @@ -15,11 +15,3 @@ {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. - -{profiles, - [{test, [ - {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.1.4"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} - ]} - ]} -]}. From 5d45325cb88ac3be9edf71816f16b4019f850d4c Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 21 Jan 2021 23:56:10 +0100 Subject: [PATCH 02/58] fix(conf): quote config values --- .github/workflows/run_cts_tests.yaml | 8 ++++---- apps/emqx_auth_http/etc/emqx_auth_http.conf | 6 +++--- apps/emqx_auth_pgsql/README.md | 2 +- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 2 +- etc/emqx.conf | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index c6f86e1ba..e7849196a 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -37,12 +37,12 @@ jobs: if: matrix.network_type == 'ipv4' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) - sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address|g\"" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address\"|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: setup if: matrix.network_type == 'ipv6' run: | server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) - sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address|g\"" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf + sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = \"$server_address\"|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -203,7 +203,7 @@ jobs: sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = postgres|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - sed -i 's|^[#[:space:]]*auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf + sed -i 's|^[#[:space:]]*auth.pgsql.ssl.enable[ \t]*=.*|auth.pgsql.ssl.enable = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf - name: setup env: @@ -263,7 +263,7 @@ jobs: run: | set -exu docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d - sed -i 's|^[#[:space:]]*auth.redis.ssl[[:space:]]*=.*|auth.redis.ssl.enable = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf + sed -i 's|^[#[:space:]]*auth.redis.ssl.enable[[:space:]]*=.*|auth.redis.ssl.enable = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf sed -i 's|^[#[:space:]]*auth.redis.ssl.cacertfile[[:space:]]*=.*|auth.redis.ssl.cacertfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf sed -i 's|^[#[:space:]]*auth.redis.ssl.certfile[[:space:]]*=.*|auth.redis.ssl.certfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf sed -i 's|^[#[:space:]]*auth.redis.ssl.keyfile[[:space:]]*=.*|auth.redis.ssl.keyfile = "/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key"|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 1ee90e982..b2f6c4280 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -18,7 +18,7 @@ auth.http.auth_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.auth_req.headers.accept = */* -auth.http.auth_req.headers.content-type = application/x-www-form-urlencoded +auth.http.auth_req.headers.content-type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters @@ -53,7 +53,7 @@ auth.http.super_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.super_req.headers.accept = */* -auth.http.super_req.headers.content-type = application/x-www-form-urlencoded +auth.http.super_req.headers.content-type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters @@ -88,7 +88,7 @@ auth.http.acl_req.method = post ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json ## ## Examples: auth.http.acl_req.headers.accept = */* -auth.http.acl_req.headers.content-type = application/x-www-form-urlencoded +auth.http.acl_req.headers.content-type = "application/x-www-form-urlencoded" ## Parameters used to construct the request body or query string parameters ## When the request method is GET, these parameters will be converted into query string parameters diff --git a/apps/emqx_auth_pgsql/README.md b/apps/emqx_auth_pgsql/README.md index 2dccd6f53..a8f5d723f 100644 --- a/apps/emqx_auth_pgsql/README.md +++ b/apps/emqx_auth_pgsql/README.md @@ -49,7 +49,7 @@ auth.pgsql.encoding = utf8 ## Whether to enable SSL connection. ## ## Value: true | false -auth.pgsql.ssl = false +auth.pgsql.ssl.enable = false ## SSL keyfile. ## diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index f2a17165e..5acb75ca1 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -103,7 +103,7 @@ auth.redis.acl_cmd = "HGETALL mqtt_acl:%u" ## CA certificate. ## ## Value: File -# auth.redis.ssl.cacertfile = path/to/your/cafile.pem +#auth.redis.ssl.cacertfile = path/to/your/cafile.pem ## Client ssl certificate. ## diff --git a/etc/emqx.conf b/etc/emqx.conf index 06539c141..4909c1221 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1717,7 +1717,7 @@ listener.ws.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for websocket connection ## ## Value: http://url eg. local http dashboard url - http://localhost:18083, http://127.0.0.1:18083 -listener.ws.external.check_origins = http://localhost:18083, http://127.0.0.1:18083 +listener.ws.external.check_origins = "http://localhost:18083, http://127.0.0.1:18083" ##-------------------------------------------------------------------- ## External WebSocket/SSL listener for MQTT Protocol @@ -2010,7 +2010,7 @@ listener.wss.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for secure websocket connection ## ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -listener.wss.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 +listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" ##-------------------------------------------------------------------- ## Modules From 7bb590f1c78840e8cc36cc52e15bdc2f4d44c835 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 22 Jan 2021 09:53:00 +0100 Subject: [PATCH 03/58] refactor(auth_redis): delete backward-compatible code in config schema --- .../priv/emqx_auth_redis.schema | 36 ++++--------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index 69a3cf388..299e1bae3 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -43,7 +43,6 @@ {datatype, string} ]}. -%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 {mapping, "auth.redis.ssl.certfile", "emqx_auth_redis.options", [ {default, ""}, {datatype, string} @@ -54,40 +53,17 @@ {datatype, string} ]}. -%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 -{mapping, "auth.redis.cafile", "emqx_auth_redis.options", [ - {default, ""}, - {datatype, string} -]}. - -%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 -{mapping, "auth.redis.certfile", "emqx_auth_redis.options", [ - {default, ""}, - {datatype, string} -]}. - -%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 -{mapping, "auth.redis.keyfile", "emqx_auth_redis.options", [ - {default, ""}, - {datatype, string} -]}. - {translation, "emqx_auth_redis.options", fun(Conf) -> - Ssl = cuttlefish:conf_get("auth.redis.ssl.enable", Conf, false), - case Ssl of + case cuttlefish:conf_get("auth.redis.ssl.enable", Conf, false) of true -> - %% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0 - Prefix = case cuttlefish:conf_get("auth.redis.ssl.cacertfile", Conf, undefined) of - undefined -> "auth.redis"; - _ -> "auth.redis.ssl" - end, - CA = cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf), - Cert = cuttlefish:conf_get(Prefix ++ ".certfile", Conf), - Key = cuttlefish:conf_get(Prefix ++ ".keyfile", Conf), + CA = cuttlefish:conf_get("auth.redis.ssl.cacertfile", Conf), + Cert = cuttlefish:conf_get("auth.redis.ssl.certfile", Conf), + Key = cuttlefish:conf_get("auth.redis.ssl.keyfile", Conf), [{options, [{ssl_options, [{cacertfile, CA}, {certfile, Cert}, {keyfile, Key}]}]}]; - _ -> [{options, []}] + false -> + [{options, []}] end end}. From 51050f702a7dfd745ee2c9141dc816e687be7792 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 22 Jan 2021 15:07:35 +0100 Subject: [PATCH 04/58] chore(config): Insert a space after config comment '#' and key --- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 5acb75ca1..997034ed3 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -98,20 +98,20 @@ auth.redis.acl_cmd = "HGETALL mqtt_acl:%u" ## Redis ssl configuration. ## ## Value: on | off -#auth.redis.ssl.enable = off +# auth.redis.ssl.enable = off ## CA certificate. ## ## Value: File -#auth.redis.ssl.cacertfile = path/to/your/cafile.pem +# auth.redis.ssl.cacertfile = path/to/your/cafile.pem ## Client ssl certificate. ## ## Value: File -#auth.redis.ssl.certfile = path/to/your/certfile +# auth.redis.ssl.certfile = path/to/your/certfile ## Client ssl keyfile. ## ## Value: File -#auth.redis.ssl.keyfile = path/to/your/keyfile +# auth.redis.ssl.keyfile = path/to/your/keyfile From fd1a53b45edc6316f2e817172b8c1f668633fba5 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 29 Jan 2021 18:00:38 +0900 Subject: [PATCH 05/58] fix(cts): fix config file path in sed --- .github/workflows/run_cts_tests.yaml | 16 ++++++++-------- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index f05cd0476..9e24a8966 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -78,10 +78,10 @@ jobs: if: matrix.connect_type == 'tls' run: | docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d - sed -i 's|^[#[:space:]]*auth.mongo.ssl[[:space:]]*=.*|auth.mongo.ssl.enable = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = "/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.ssl.enable[[:space:]]*=.*|auth.mongo.ssl.enable = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf + sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = \"/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem\"|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf - name: setup env: MONGO_TAG: ${{ matrix.mongo_tag }} @@ -132,10 +132,10 @@ jobs: if: matrix.connect_type == 'tls' run: | docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d - sed -i 's|^[#[:space:]]*auth.mysql.ssl[[:space:]]*=.*|auth.mysql.ssl.enable = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - sed -i 's|^[#[:space:]]*auth.mysql.ssl.cacertfile[[:space:]]*=.*|auth.mysql.ssl.cacertfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - sed -i 's|^[#[:space:]]*auth.mysql.ssl.certfile[[:space:]]*=.*|auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - sed -i 's|^[#[:space:]]*auth.mysql.ssl.keyfile[[:space:]]*=.*|auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.ssl.enable[[:space:]]*=.*|auth.mysql.ssl.enable = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.ssl.cacertfile[[:space:]]*=.*|auth.mysql.ssl.cacertfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.ssl.certfile[[:space:]]*=.*|auth.mysql.ssl.certfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf + sed -i 's|^[#[:space:]]*auth.mysql.ssl.keyfile[[:space:]]*=.*|auth.mysql.ssl.keyfile = \"/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem\"|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf - name: setup env: MYSQL_TAG: ${{ matrix.mysql_tag }} diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 814309e1a..42e04b450 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -53,7 +53,7 @@ auth.mongo.database = mqtt ## Whether to enable SSL connection. ## ## Value: on | off -## auth.mongo.ssl = off +## auth.mongo.ssl.enable = off ## SSL keyfile. ## From a6e532779b668fac615b8d7fc430f9cb3c870549 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 29 Jan 2021 18:01:00 +0900 Subject: [PATCH 06/58] chore(mongo): quote example --- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 42e04b450..8ae68f3e7 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -16,7 +16,7 @@ auth.mongo.type = single ## ## Value: String ## -## Examples: 127.0.0.1:27017,127.0.0.2:27017... +## Examples: "127.0.0.1:27017,127.0.0.2:27017,..." auth.mongo.server = "127.0.0.1:27017" ## MongoDB pool size From 3863b7f11616c5c8601d9877089f15a6c79e9edd Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 12 Feb 2021 11:13:19 +0100 Subject: [PATCH 07/58] fix(config): Quote config string values --- apps/emqx_bridge_mqtt/README.md | 36 +++++++++++------------ apps/emqx_bridge_mqtt/docs/guide.rst | 44 ++++++++++++++-------------- etc/emqx.conf | 8 ++--- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md index 456fae584..812645627 100644 --- a/apps/emqx_bridge_mqtt/README.md +++ b/apps/emqx_bridge_mqtt/README.md @@ -53,13 +53,13 @@ The following is the basic configuration of RPC bridging. A simplest RPC bridgin ``` ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection -bridge.mqtt.emqx2.address = emqx2@192.168.1.2 +bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" ## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# +bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## bridged mountpoint -bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ +bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ``` If the messages received by the local node emqx1 matches the topic `sersor1/#` or `sensor2/#`, these messages will be forwarded to the `sensor1/#` or `sensor2/#` topic of the remote node emqx2. @@ -82,66 +82,66 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co ``` ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection -bridge.mqtt.emqx2.address = 192.168.1.2:1883 +bridge.mqtt.emqx2.address = "192.168.1.2:1883" ## Bridged Protocol Version ## Enumeration value: mqttv3 | mqttv4 | mqttv5 -bridge.mqtt.emqx2.proto_ver = mqttv4 +bridge.mqtt.emqx2.proto_ver = "mqttv4" ## mqtt client's clientid -bridge.mqtt.emqx2.clientid = bridge_emq +bridge.mqtt.emqx2.clientid = "bridge_emq" ## mqtt client's clean_start field ## Note: Some MQTT Brokers need to set the clean_start value as `true` bridge.mqtt.emqx2.clean_start = true ## mqtt client's username field -bridge.mqtt.emqx2.username = user +bridge.mqtt.emqx2.username = "user" ## mqtt client's password field -bridge.mqtt.emqx2.password = passwd +bridge.mqtt.emqx2.password = "passwd" ## Whether the mqtt client uses ssl to connect to a remote serve or not bridge.mqtt.emqx2.ssl = off ## CA Certificate of Client SSL Connection (PEM format) -bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem +bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" ## SSL certificate of Client SSL connection -bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem +bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" ## Key file of Client SSL connection -bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem +bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" ## SSL encryption -bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 +bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" ## TTLS PSK password ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time ## ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Client's heartbeat interval bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version -bridge.mqtt.emqx2.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +bridge.mqtt.emqx2.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" ## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# +bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## Bridged mountpoint -bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ +bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1 +bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.1.qos = 1 ## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2 +bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.2.qos = 1 diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst index 47235aa7d..a1c2a9126 100644 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ b/apps/emqx_bridge_mqtt/docs/guide.rst @@ -39,7 +39,7 @@ In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguish .. code-block:: ## Bridge address: node name for local bridge, host:port for remote. - bridge.mqtt.aws.address = 127.0.0.1:1883 + bridge.mqtt.aws.address = "127.0.0.1:1883" This configuration declares a bridge named ``aws`` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. @@ -69,13 +69,13 @@ The following is the basic configuration of RPC bridging. A simplest RPC bridgin .. code-block:: ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection - bridge.mqtt.emqx2.address = emqx2@192.168.1.2 + bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# + bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## bridged mountpoint - bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ + bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" If the messages received by the local node emqx1 matches the topic ``sersor1/#`` or ``sensor2/#``\ , these messages will be forwarded to the ``sensor1/#`` or ``sensor2/#`` topic of the remote node emqx2. @@ -86,10 +86,10 @@ If the messages received by the local node emqx1 matches the topic ``sersor1/#`` Limitations of RPC bridging: -#. +#. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; -#. +#. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. EMQ X MQTT Bridge Configuration @@ -102,66 +102,66 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co .. code-block:: ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection - bridge.mqtt.emqx2.address = 192.168.1.2:1883 + bridge.mqtt.emqx2.address = "192.168.1.2:1883" ## Bridged Protocol Version ## Enumeration value: mqttv3 | mqttv4 | mqttv5 - bridge.mqtt.emqx2.proto_ver = mqttv4 + bridge.mqtt.emqx2.proto_ver = "mqttv4" ## mqtt client's clientid - bridge.mqtt.emqx2.clientid = bridge_emq + bridge.mqtt.emqx2.clientid = "bridge_emq" ## mqtt client's clean_start field ## Note: Some MQTT Brokers need to set the clean_start value as `true` bridge.mqtt.emqx2.clean_start = true ## mqtt client's username field - bridge.mqtt.emqx2.username = user + bridge.mqtt.emqx2.username = "user" ## mqtt client's password field - bridge.mqtt.emqx2.password = passwd + bridge.mqtt.emqx2.password = "passwd" ## Whether the mqtt client uses ssl to connect to a remote serve or not bridge.mqtt.emqx2.ssl = off ## CA Certificate of Client SSL Connection (PEM format) - bridge.mqtt.emqx2.cacertfile = etc/certs/cacert.pem + bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" ## SSL certificate of Client SSL connection - bridge.mqtt.emqx2.certfile = etc/certs/client-cert.pem + bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" ## Key file of Client SSL connection - bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem + bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" ## TTLS PSK password ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time ## ## See 'https://tools.ietf.org/html/rfc4279#section-2'. - ## bridge.mqtt.emqx2.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA + ## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" ## Client's heartbeat interval bridge.mqtt.emqx2.keepalive = 60s ## Supported TLS version - bridge.mqtt.emqx2.tls_versions = tlsv1.2 + bridge.mqtt.emqx2.tls_versions = "tlsv1.2" ## SSL encryption - bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/# + bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" ## Bridged mountpoint - bridge.mqtt.emqx2.mountpoint = bridge/emqx2/${node}/ + bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.1.topic = cmd/topic1 + bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.1.qos = 1 ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.2.topic = cmd/topic2 + bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" ## Subscription qos for bridging bridge.mqtt.emqx2.subscription.2.qos = 1 @@ -190,7 +190,7 @@ The bridge of EMQ X has a message caching mechanism. The caching mechanism is ap bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB ## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. - bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/ + bridge.mqtt.emqx2.queue.replayq_dir = "data/emqx_emqx2_bridge/" ## Replayq data segment size bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB diff --git a/etc/emqx.conf b/etc/emqx.conf index 43b402d98..0d76f111c 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -184,12 +184,12 @@ cluster.autoclean = 5m ## Value: @ ## ## Default: emqx@127.0.0.1 -node.name = emqx@127.0.0.1 +node.name = "emqx@127.0.0.1" ## Cookie for distributed node communication. ## ## Value: String -node.cookie = emqxsecretcookie +node.cookie = "emqxsecretcookie" ## Data dir for the node ## @@ -1564,7 +1564,7 @@ listener.ws.external.access.1 = "allow all" ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.ws.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.ws.external.supported_protocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. @@ -1785,7 +1785,7 @@ listener.wss.external.access.1 = "allow all" ## Supported subprotocols ## ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.wss.external.supported_protocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.wss.external.supported_protocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" ## Enable the Proxy Protocol V1/2 support. ## From f5f79af41859a88612028ea733de2ed14b618078 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 12 Feb 2021 13:12:02 +0100 Subject: [PATCH 08/58] fix(boot): allow quoted node name and cookie --- bin/emqx | 4 ++-- bin/emqx_ctl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/emqx b/bin/emqx index 025f08ce5..ed7d81a67 100755 --- a/bin/emqx +++ b/bin/emqx @@ -255,7 +255,7 @@ if [ -z "$NAME_ARG" ]; then # check if there is a node running, inspect its name # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}') - [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) + [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2- | tr -d '"') if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." echoerr " -sname is not supported." @@ -280,7 +280,7 @@ if [ -z "$COOKIE_ARG" ]; then # check if there is a node running, steal its cookie # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}') - [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) + [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2- | tr -d '"') if [ -z "$COOKIE" ]; then echoerr "vm.args needs to have a -setcookie parameter." echoerr "please check $RUNNER_ETC_DIR/emqx.conf" diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 961832c86..30f83b6a8 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -37,7 +37,7 @@ if [ -z "$NAME_ARG" ]; then # check if there is a node running, inspect its name # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}') - [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) + [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2- | tr -d '"') if [ -z "$NODENAME" ]; then echoerr "vm.args needs to have a -name parameter." echoerr " -sname is not supported." @@ -58,7 +58,7 @@ if [ -z "$COOKIE_ARG" ]; then # check if there is a node running, steal its cookie # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}') - [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-) + [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2- | tr -d '"') if [ -z "$COOKIE" ]; then echoerr "vm.args needs to have a -setcookie parameter." echoerr "please check $RUNNER_ETC_DIR/emqx.conf" From 469c7ad43af225c370b4074075febdcc04cc6a9a Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 19 Feb 2021 13:15:11 +0900 Subject: [PATCH 09/58] chore(config): rm redundant quotes because they are now processed in cuttlefish --- .ci/fvt_tests/docker-compose.yaml | 4 ++-- deploy/charts/emqx/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml index 60d4bd64b..22d48bef7 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/fvt_tests/docker-compose.yaml @@ -8,7 +8,7 @@ services: - "EMQX_NAME=emqx" - "EMQX_HOST=node1.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\"" + - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" command: @@ -35,7 +35,7 @@ services: - "EMQX_NAME=emqx" - "EMQX_HOST=node2.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - - "EMQX_CLUSTER__STATIC__SEEDS=\"emqx@node1.emqx.io, emqx@node2.emqx.io\"" + - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" command: diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index b2a687f78..45b966c3b 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -42,7 +42,7 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: - EMQX_CLUSTER__K8S__APISERVER: \"https://kubernetes.default.svc:443\" + EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note：Hostname is only supported after v4.0-rc.2 From 62f1bd8120ac9ccb3ead642ee59fc90bbb1cb5fc Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 7 Apr 2021 20:29:53 +0900 Subject: [PATCH 10/58] chore(rebar.config): use z8674558/cuttlefish temporalily --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 27ab75ecb..17af3dcea 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {branch, "hocon"}}} + , {cuttlefish, {git, "https://github.com/z8674558/cuttlefish", {branch, "fix-eunit"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} From 490e6198ba2182682030d460f853846d298286c6 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 7 Apr 2021 20:39:11 +0900 Subject: [PATCH 11/58] fix(apps): typos on schema --- apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema | 2 +- apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema index 150990818..cd8c03015 100644 --- a/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema +++ b/apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema @@ -134,7 +134,7 @@ SslOpts = fun(Prefix) -> Verify = case cuttlefish:conf_get(Prefix ++ ".verify", Conf, false) of true -> verify_peer; - flase -> verify_none + false -> verify_none end, Filter([{verify, Verify}, {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of diff --git a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema index 2f7fd3886..2be9b0670 100644 --- a/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema +++ b/apps/emqx_auth_pgsql/priv/emqx_auth_pgsql.schema @@ -101,7 +101,7 @@ SslOpts = fun(Prefix) -> Verify = case cuttlefish:conf_get(Prefix ++ ".verify", Conf, false) of true -> verify_peer; - flase -> verify_none + false -> verify_none end, Filter([{keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, From 70a54500265706d80bdc8dfc0da7f17da57864a5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 9 Apr 2021 22:54:15 +0200 Subject: [PATCH 12/58] feat(log): rfc3339 date-time format in logs e.g. default: 2021-04-09T22:53:50.183617+02:00 --- etc/emqx.conf | 7 ++++++ priv/emqx.schema | 33 +++++++++++++++++++++++++++- src/emqx_logger_formatter.erl | 33 ++++++++++++++++++++-------- test/emqx_logger_formatter_SUITE.erl | 19 ++++++---------- 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 2df17c5bc..a3aaee1f9 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -425,6 +425,13 @@ log.to = file ## Default: warning log.level = warning +## Timezone offset to display in logs +## Value: +## - "system" use system zone +## - "utc" for Universal Coordinated Time (UTC) +## - "+hh:mm" or "-hh:mm" for a specified offset +log.time_offset = system + ## The dir for log files. ## ## Value: Folder diff --git a/priv/emqx.schema b/priv/emqx.schema index f0c656993..a5010ce8d 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -465,6 +465,15 @@ end}. {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} ]}. +%% @doc Timezone offset to display in logs, +%% "system" use system time zone +%% "utc" for Universal Coordinated Time (UTC) +%% "+hh:mm" or "-hh:mm" for a specified offset +{mapping, "log.time_offset", "kernel.logger", [ + {default, "system"}, + {datatype, string} +]}. + {mapping, "log.primary_log_level", "kernel.logger_level", [ {default, warning}, {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} @@ -584,6 +593,26 @@ end}. {translation, "kernel.logger", fun(Conf) -> LogTo = cuttlefish:conf_get("log.to", Conf), LogLevel = cuttlefish:conf_get("log.level", Conf), + LogTimeoffset = + case cuttlefish:conf_get("log.time_offset", Conf) of + "system" -> ""; + "utc" -> "0"; + [S, H1, H2, $:, M1, M2] = HHMM -> + (S =:= $+ orelse S =:= $-) andalso + try + begin + H = list_to_integer([H1, H2]), + M = list_to_integer([M1, M2]), + H >=0 andalso H =< 14 andalso + M >= 0 andalso M =< 59 + end + catch + _ : _ -> + error({"invalid_log_time_offset", HHMM}) + end andalso HHMM; + Other -> + error({"invalid_log_time_offset", Other}) + end, LogType = case cuttlefish:conf_get("log.rotation.enable", Conf) of true -> wrap; false -> halt @@ -603,7 +632,9 @@ end}. [peername," "], []}]}, msg,"\n"], - chars_limit => CharsLimit}}, + chars_limit => CharsLimit, + time_offset => LogTimeoffset + }}, {BustLimitOn, {MaxBurstCount, TimeWindow}} = case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of ["disabled"] -> {false, {20000, 1000}}; diff --git a/src/emqx_logger_formatter.erl b/src/emqx_logger_formatter.erl index 2528a63b5..b8642271d 100644 --- a/src/emqx_logger_formatter.erl +++ b/src/emqx_logger_formatter.erl @@ -246,14 +246,10 @@ truncate(String,Size) -> String end. -%% Convert microseconds-timestamp into local datatime string in milliseconds -format_time(SysTime,#{}) - when is_integer(SysTime) -> - Ms = SysTime rem 1000000 div 1000, - {Date, _Time = {H, Mi, S}} = calendar:system_time_to_local_time(SysTime, microsecond), - format_time({Date, {H, Mi, S, Ms}}). -format_time({{Y, M, D}, {H, Mi, S, Ms}}) -> - io_lib:format("~b-~2..0b-~2..0b ~2..0b:~2..0b:~2..0b.~3..0b", [Y, M, D, H, Mi, S, Ms]). +format_time(SysTime, Config) when is_integer(SysTime) -> + Offset = maps:get(time_offset, Config, ""), + calendar:system_time_to_rfc3339(SysTime, [{unit,microsecond}, + {offset,Offset}]). format_mfa({M,F,A},_) when is_atom(M), is_atom(F), is_integer(A) -> atom_to_list(M)++":"++atom_to_list(F)++"/"++integer_to_list(A); @@ -313,12 +309,31 @@ do_check_config([{template,T}|Config]) -> ok -> do_check_config(Config); error -> {error,{invalid_formatter_template,?MODULE,T}} end; - +do_check_config([{time_offset, Offset} | Config]) -> + case lists:member(Offset, ["", "0", "Z", "z"]) orelse check_time_offset(Offset) of + true -> do_check_config(Config); + error -> {error, {time_offset, ?MODULE, Offset}} + end; do_check_config([C|_]) -> {error,{invalid_formatter_config,?MODULE,C}}; do_check_config([]) -> ok. +check_time_offset([S, H1, H2, $:, M1, M2]) -> + (S =:= $+ orelse S =:= $-) andalso + try + begin + H = list_to_integer([H1, H2]), + M = list_to_integer([M1, M2]), + H >=0 andalso H =< 14 andalso + M >= 0 andalso M =< 59 + end + catch + _ : _ -> + error + end; +check_time_offset(_) -> error. + check_limit(L) when is_integer(L), L>0 -> ok; check_limit(unlimited) -> diff --git a/test/emqx_logger_formatter_SUITE.erl b/test/emqx_logger_formatter_SUITE.erl index 3d7469dca..a495809b5 100644 --- a/test/emqx_logger_formatter_SUITE.erl +++ b/test/emqx_logger_formatter_SUITE.erl @@ -75,7 +75,7 @@ all() -> default(_Config) -> String1 = format(info,{"~p",[term]},#{},#{}), ct:log(String1), - ?assertMatch([_Date, _Time,"info:","term\n"], string:lexemes(String1," ")), + ?assertMatch([_DateTime,"info:","term\n"], string:lexemes(String1, " ")), Time = timestamp(), ExpectedTimestamp = default_time_format(Time), @@ -581,8 +581,8 @@ update_config(_Config) -> ct:log("lines1: ~p", [Lines1]), ct:log("c1: ~p",[C1]), [Line1 | _] = Lines1, - [_Date, WithOutDate1] = string:split(Line1," "), - [_, "notice: "++D1] = string:split(WithOutDate1," "), + [_DateTime1, WithOutDate1] = string:split(Line1, " "), + ["notice:", D1] = string:split(WithOutDate1, " "), ?assert(length(D1)<1000), ?assertMatch(#{chars_limit := unlimited}, C1), @@ -591,8 +591,8 @@ update_config(_Config) -> ct:log("Lines5: ~p", [Lines5]), ct:log("c5: ~p",[C5]), [Line5 | _] = Lines5, - [_Date, WithOutDate5] = string:split(Line5," "), - [_, "error: "++_D5] = string:split(WithOutDate5," "), + [_DateTime2, WithOutDate5] = string:split(Line5, " "), + ["error:", _D5] = string:split(WithOutDate5, " "), ?assertMatch({error,{invalid_formatter_config,bad}}, logger:update_formatter_config(?MODULE,bad)), @@ -616,13 +616,8 @@ format(Log,Config) -> lists:flatten(emqx_logger_formatter:format(Log,Config)). default_time_format(SysTime) when is_integer(SysTime) -> - Ms = SysTime rem 1000000 div 1000, - {Date, _Time = {H, Mi, S}} = calendar:system_time_to_local_time(SysTime, microsecond), - default_time_format({Date, {H, Mi, S, Ms}}); -default_time_format({{Y, M, D}, {H, Mi, S, Ms}}) -> - io_lib:format("~b-~2..0b-~2..0b ~2..0b:~2..0b:~2..0b.~3..0b", [Y, M, D, H, Mi, S, Ms]); -default_time_format({{Y, M, D}, {H, Mi, S}}) -> - io_lib:format("~b-~2..0b-~2..0b ~2..0b:~2..0b:~2..0b", [Y, M, D, H, Mi, S]). + calendar:system_time_to_rfc3339(SysTime, [{unit,microsecond}, + {offset,""}]). integer(Str) -> is_integer(list_to_integer(Str)). From ec69e51386b96c96fbac68aef1ab05d027207c0f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 21:46:04 +0200 Subject: [PATCH 13/58] build: pin cuttlefish v4.0.0 (hocon) --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 12bb8eb98..77e61ab35 100644 --- a/rebar.config +++ b/rebar.config @@ -44,7 +44,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {branch, "hocon"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} From 7c29884ed02b164f1a7f95eb29fd76f8022cc1b2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 7 May 2021 22:33:39 +0200 Subject: [PATCH 14/58] build: zip for macos previsouly zip was built before pkg (as makefile dependency) since pkg's only depend on rebar3 release tarball (not .zip) makefile was updated but forgot to update the pkg build for macos --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index af7ff9865..4d2997b84 100755 --- a/build +++ b/build @@ -134,7 +134,7 @@ case "$ARTIFACT" in ;; pkg) if [ -z "${PKGERDIR:-}" ]; then - # zip should have been built as a Makefile dependency + make_zip log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi From b288710429f56d05d6304a8805c15d3dd203b60c Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 11:54:40 +0800 Subject: [PATCH 15/58] chore(CI): update build workflows --- .ci/build_packages/Dockerfile | 2 + .github/workflows/build_packages.yaml | 10 +-- .github/workflows/build_slim_packages.yaml | 73 +++++++++++++++++++++- build | 1 - 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/.ci/build_packages/Dockerfile b/.ci/build_packages/Dockerfile index aa45c970b..792acddf5 100644 --- a/.ci/build_packages/Dockerfile +++ b/.ci/build_packages/Dockerfile @@ -7,6 +7,8 @@ COPY . /emqx WORKDIR /emqx +RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump + RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump RUN /emqx/.ci/build_packages/tests.sh diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 697820299..f1ddef88a 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -123,6 +123,8 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + erl_otp: + - 23.2.7.2 exclude: - profile: emqx-edge @@ -141,12 +143,12 @@ jobs: - name: build erlang timeout-minutes: 60 run: | - kerl build 23.2.7 - kerl install 23.2.7 $HOME/.kerl/23.2.7 + kerl build ${{ matrix.erl_otp }} + kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - name: build run: | - . $HOME/.kerl/23.2.7/activate - make -C source ${{ matrix.profile }}-pkg + . $HOME/.kerl/${{ matrix.erl_otp }}/activate + make -C source ${{ matrix.profile }}-zip - name: test run: | cd source diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a78d93a56..aae5cfd18 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -1,8 +1,12 @@ name: Build slim packages on: - - pull_request - - workflow_dispatch + push: + tags: + - v* + - e* + pull_request: + workflow_dispatch: jobs: build: @@ -30,7 +34,9 @@ jobs: else echo "EMQX_NAME=emqx" >> $GITHUB_ENV fi - - name: build packages + - name: build zip packages + run: make ${EMQX_NAME}-zip + - name: build deb/rpm packages run: make ${EMQX_NAME}-pkg - name: pakcages test run: | @@ -40,3 +46,64 @@ jobs: with: name: ${{ matrix.os }} path: _packages/**/*.zip + + mac: + runs-on: macos-10.15 + + strategy: + matrix: + erl_otp: + - 23.2.7.2 + + steps: + - uses: actions/checkout@v1 + - name: prepare + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token + echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV + else + echo "EMQX_NAME=emqx" >> $GITHUB_ENV + fi + - name: prepare + run: | + brew install curl zip unzip gnu-sed kerl unixodbc freetds + echo "/usr/local/bin" >> $GITHUB_PATH + git config --global credential.helper store + - name: build erlang + timeout-minutes: 60 + run: | + kerl build ${{ matrix.erl_otp }} + kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} + - name: build + run: | + . $HOME/.kerl/${{ matrix.erl_otp }}/activate + make ${EMQX_NAME}-zip + - name: test + run: | + pkg_name=$(basename _packages/emqx/emqx-*.zip) + unzip _packages/emqx/$pkg_name + gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 + ready='no' + for i in {1..10}; do + if curl -fs 127.0.0.1:18083 > /dev/null; then + ready='yes' + break + fi + sleep 1 + done + if [ "$ready" != "yes" ]; then + echo "Timed out waiting for emqx to be ready" + cat emqx/log/erlang.log.1 + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx stop + rm -rf emqx + - uses: actions/upload-artifact@v1 + with: + name: macos + path: _packages/emqx/. diff --git a/build b/build index 4d2997b84..be7813e66 100755 --- a/build +++ b/build @@ -134,7 +134,6 @@ case "$ARTIFACT" in ;; pkg) if [ -z "${PKGERDIR:-}" ]; then - make_zip log "Skipped making deb/rpm package for $SYSTEM" exit 0 fi From cc6f43aea7ae6c36cc7b4ad4c15a9ce112230159 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 14:11:49 +0800 Subject: [PATCH 16/58] fix(import): fix import bridge mqtt test cases --- ...x_bridge_mqtt_data_export_import_SUITE.erl | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index 2609e1006..7cf7d19cf 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -28,24 +28,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> - ok = ekka_mnesia:start(), - ok = emqx_rule_registry:mnesia(boot), - ok = emqx_rule_engine:load_providers(), - emqx_ct_helpers:start_apps([emqx_web_hook, - emqx_bridge_mqtt, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard]), + emqx_ct_helpers:start_apps([emqx_bridge_mqtt, emqx_rule_engine]), Cfg. end_per_suite(Cfg) -> - emqx_ct_helpers:stop_apps([emqx_dashboard, - emqx_management, - emqx_modules, - emqx_rule_engine, - emqx_bridge_mqtt, - emqx_web_hook]), + emqx_ct_helpers:stop_apps([emqx_management, emqx_rule_engine]), Cfg. get_data_path() -> @@ -53,8 +40,8 @@ get_data_path() -> import(FilePath, Version) -> ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + timer:sleep(500), lists:foreach(fun(#resource{id = Id, config = Config} = _Resource) -> - timer:sleep(2000), case Id of <<"bridge">> -> test_utils:resource_is_alive(Id), @@ -74,19 +61,23 @@ import(FilePath, Version) -> t_import420(_) -> import("420.json", 420), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import430(_) -> import("430.json", 430), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import409(_) -> import("409.json", 409), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_import415(_) -> import("415.json", 415), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). handle_config(Config, 420, bridge) -> @@ -126,27 +117,33 @@ handle_config(_, _, _) -> ok. t_importee4010(_) -> import("ee4010.json", ee4010), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee410(_) -> import("ee410.json", ee410), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee411(_) -> import("ee411.json", ee411), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee420(_) -> import("ee420.json", ee420), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee425(_) -> import("ee425.json", ee425), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). t_importee430(_) -> import("ee430.json", ee430), - {ok, _} = emqx_mgmt_data_backup:export(). + {ok, _} = emqx_mgmt_data_backup:export(), + remove_resources(). %%-------------------------------------------------------------------- %% handle_config @@ -176,3 +173,11 @@ handle_config(Config, ee430, rpc) -> handle_config(Config, ee435, Id) -> handle_config(Config, ee430, Id). -endif. + +remove_resources() -> + timer:sleep(500), + lists:foreach(fun(#resource{id = Id}) -> + emqx_rule_registry:remove_resource(Id), + emqx_rule_registry:remove_resource_params(Id) + end, emqx_rule_registry:get_resources()), + timer:sleep(500). \ No newline at end of file From a58357cfe3a5a41bb9f5287bb5984673ff0181c4 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 16:32:17 +0800 Subject: [PATCH 17/58] feat(rule_engine): rule sql add null function --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 4 ++++ apps/emqx_rule_engine/src/emqx_rule_utils.erl | 1 + 2 files changed, 5 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 4f8851d40..a96ee7a62 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -35,6 +35,7 @@ , contains_topic/3 , contains_topic_match/2 , contains_topic_match/3 + , null/0 ]). %% Arithmetic Funcs @@ -300,6 +301,9 @@ find_topic_filter(Filter, TopicFilters, Func) -> throw:Result -> Result end. +null() -> + undefined. + %%------------------------------------------------------------------------------ %% Arithmetic Funcs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/src/emqx_rule_utils.erl b/apps/emqx_rule_engine/src/emqx_rule_utils.erl index ff7ee0304..3791b1386 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_utils.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_utils.erl @@ -228,6 +228,7 @@ tcp_connectivity(Host, Port, Timeout) -> unwrap(<<"${", Val/binary>>) -> binary:part(Val, {0, byte_size(Val)-1}). +sql_data(undefined) -> null; sql_data(List) when is_list(List) -> List; sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; From 7ef173f2c279d9d14d6df267ca53f7103177118e Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 8 May 2021 20:47:46 +0800 Subject: [PATCH 18/58] chore(slim): fix build silm macosx fail --- .github/workflows/build_slim_packages.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index aae5cfd18..f3362cd44 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -83,8 +83,8 @@ jobs: make ${EMQX_NAME}-zip - name: test run: | - pkg_name=$(basename _packages/emqx/emqx-*.zip) - unzip _packages/emqx/$pkg_name + pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) + unzip _packages/${EMQX_NAME}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' From 0abdde0badd3a460eeb27cf17fef5089d3089f2e Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 13:59:24 +0800 Subject: [PATCH 19/58] chore(CI): use cache for mac --- .github/workflows/build_packages.yaml | 9 ++++++++- .github/workflows/build_slim_packages.yaml | 13 ++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f1ddef88a..fde6795b1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -137,10 +137,17 @@ jobs: run: unzip -q source.zip - name: prepare run: | + brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store + - uses: actions/cache@v2 + id: cache + with: + path: ~/.kerl + key: erl${{ matrix.erl_otp }}-macos10.15 - name: build erlang + if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 run: | kerl build ${{ matrix.erl_otp }} @@ -153,7 +160,7 @@ jobs: run: | cd source pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip _packages/${{ matrix.profile }}/$pkg_name + unzip -q _packages/${{ matrix.profile }}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index f3362cd44..573b84e1b 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -69,10 +69,17 @@ jobs: fi - name: prepare run: | + brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH git config --global credential.helper store + - uses: actions/cache@v2 + id: cache + with: + path: ~/.kerl + key: erl${{ matrix.erl_otp }}-macos10.15 - name: build erlang + if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 run: | kerl build ${{ matrix.erl_otp }} @@ -84,7 +91,7 @@ jobs: - name: test run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip _packages/${EMQX_NAME}/$pkg_name + unzip -q _packages/${EMQX_NAME}/$pkg_name gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -103,7 +110,7 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v2 with: name: macos - path: _packages/emqx/. + path: _packages/**/*.zip From 3a2e177be5458e25b3f201ac238c0bc3e0c611ab Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 11:11:56 +0200 Subject: [PATCH 20/58] fix(logging): add line-break after each JSON log --- src/emqx.app.src | 2 +- src/emqx.appup.src | 7 ++++++- src/emqx_logger_jsonfmt.erl | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/emqx.app.src b/src/emqx.app.src index a9ae81cac..449ffd311 100644 --- a/src/emqx.app.src +++ b/src/emqx.app.src @@ -1,7 +1,7 @@ {application, emqx, [{id, "emqx"}, {description, "EMQ X"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 87ca0c005..5b6a5a4d5 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,9 +1,14 @@ %% -*-: erlang -*- {VSN, - [ + [ {"4.3.0", + [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ], [ + {"4.3.0", + [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ] }. diff --git a/src/emqx_logger_jsonfmt.erl b/src/emqx_logger_jsonfmt.erl index 7d0a1a328..31e4c2fda 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/src/emqx_logger_jsonfmt.erl @@ -48,7 +48,7 @@ -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta}, Config0) when is_map(Config0) -> Config = add_default_config(Config0), - format(Msg, Meta#{level => Level}, Config). + [format(Msg, Meta#{level => Level}, Config) , "\n"]. format(Msg, Meta, Config) -> Data0 = From cb5a21ceff9c1496380aa6a323fd77723c656749 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 16:05:51 +0200 Subject: [PATCH 21/58] chore(nodetool): ensure correct lib dirs Prioro to this change, ekka lib dir was added by inspecting the ekka lib version in persisted RELEASES file. However, this file may not be the true version if a hot upgrade failed in the middle. In this fix, it reads the per-version emqx.rel file content to load lib paths. --- bin/emqx | 3 ++- bin/nodetool | 20 ++++++++++++++++++++ data/emqx_vars | 1 - 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/bin/emqx b/bin/emqx index da1ac8990..fd6986614 100755 --- a/bin/emqx +++ b/bin/emqx @@ -174,9 +174,10 @@ relx_gen_id() { # Control a node relx_nodetool() { command="$1"; shift + export RUNNER_ROOT_DIR + export REL_VSN ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \ - ERL_LIBS="${LIB_EKKA_DIR}:${ERL_LIBS:-}" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ -setcookie "$COOKIE" "$command" "$@" } diff --git a/bin/nodetool b/bin/nodetool index e67ef4790..5ab5ec69a 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -19,6 +19,7 @@ main(Args) -> ok end end, + ok = add_libs_dir(), ok = do_with_halt(Args, "mnesia_dir", fun create_mnesia_dir/2), ok = do_with_halt(Args, "chkconfig", fun("-config", X) -> chkconfig(X) end), ok = do_with_halt(Args, "chkconfig", fun chkconfig/1), @@ -288,3 +289,22 @@ join([], Sep) when is_list(Sep) -> []; join([H|T], Sep) -> H ++ lists:append([Sep ++ X || X <- T]). + +add_libs_dir() -> + [_ | _] = RootDir = os:getenv("RUNNER_ROOT_DIR"), + RelFile = filename:join([RootDir, "releases", + os:getenv("REL_VSN"), + "emqx.rel" + ]), + {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} = file:consult(RelFile), + lists:foreach( + fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); + ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) + end, Libs). + +add_lib_dir(RootDir, Name, Vsn) -> + LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), + case code:add_pathz(LibDir) of + true -> ok; + {error, _} -> error(LibDir) + end. diff --git a/data/emqx_vars b/data/emqx_vars index 8081badbe..453bd618f 100644 --- a/data/emqx_vars +++ b/data/emqx_vars @@ -13,7 +13,6 @@ RUNNER_LIB_DIR="{{ runner_lib_dir }}" RUNNER_ETC_DIR="{{ runner_etc_dir }}" RUNNER_DATA_DIR="{{ runner_data_dir }}" RUNNER_USER="{{ runner_user }}" -LIB_EKKA_DIR="${RUNNER_LIB_DIR}/ekka-$(grep ekka "${RUNNER_ROOT_DIR}/releases/RELEASES" | awk -F '\"' '{print $2}')" EMQX_LICENSE_CONF='' export EMQX_DESCRIPTION='{{ emqx_description }}' From efe0c2fe1a3dab36745264aac937d6dc1a86ad37 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Mon, 10 May 2021 15:08:36 +0900 Subject: [PATCH 22/58] fix(docker-entrypoint): do not mv loaded_plugins file, copy it by cat --- .ci/docker-compose-file/conf.cluster.env | 1 + .github/workflows/run_fvt_tests.yaml | 7 +++++++ deploy/docker/docker-entrypoint.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.ci/docker-compose-file/conf.cluster.env b/.ci/docker-compose-file/conf.cluster.env index 16fb4733e..d8294a785 100644 --- a/.ci/docker-compose-file/conf.cluster.env +++ b/.ci/docker-compose-file/conf.cluster.env @@ -4,3 +4,4 @@ EMQX_CLUSTER__STATIC__SEEDS="emqx@node1.emqx.io, emqx@node2.emqx.io" EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on EMQX_LOG__LEVEL=debug +EMQX_LOADED_PLUGINS=emqx_sn diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 68aec59a6..8f65d022e 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -46,6 +46,13 @@ jobs: echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; sleep 5; done + - name: verify EMQX_LOADED_PLUGINS override working + run: | + expected="{emqx_sn, true}." + output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) + if [ "$expected" != "$output" ]; then + exit 1 + fi - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh; then diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index a95aa50ee..16b6cb077 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -98,7 +98,7 @@ fill_tuples() { local elements=${*:2} for var in $elements; do if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then - sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && mv tmpfile "$file" + sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" elif grep -q "$var\s*\." "$file"; then # backward compatible. sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file" From 96639b8d9efc2d2cf7c2d19c8e66250d3c9a254e Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sun, 9 May 2021 12:10:21 +0200 Subject: [PATCH 23/58] chore(config): do not expose manual rpc peer port config when using `manual`, all nodes in the cluster should agree to the same port number, so there is no need to make peer port number for clients. --- etc/emqx.conf | 17 ++++++----------- priv/emqx.schema | 16 +++++++++------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index dca8811a9..7b27f100c 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -331,25 +331,20 @@ rpc.async_batch_size = 256 ## Defaults to `stateless`. rpc.port_discovery = stateless -## TCP server port for RPC. +## TCP port number for RPC server to listen on. ## ## Only takes effect when `rpc.port_discovery` = `manual`. ## +## NOTE: All nodes in the cluster should agree to this same config. +## ## Value: Port [1024-65535] #rpc.tcp_server_port = 5369 -## TCP port for outgoing RPC connections. -## -## Only takes effect when `rpc.port_discovery` = `manual`. -## -## Value: Port [1024-65535] -#rpc.tcp_client_port = 5369 - ## Number of outgoing RPC connections. ## -## Value: Interger [1-256] -## Defaults to NumberOfCPUSchedulers / 2 -#rpc.tcp_client_num = 1 +## Value: Interger [0-256] +## Defaults to NumberOfCPUSchedulers / 2 when set to 0 +#rpc.tcp_client_num = 0 ## RCP Client connect timeout. ## diff --git a/priv/emqx.schema b/priv/emqx.schema index 1d145d8f5..e7a96abcb 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -367,13 +367,7 @@ end}. {datatype, integer} ]}. -%% Default TCP port for outgoing connections -{mapping, "rpc.tcp_client_port", "gen_rpc.tcp_client_port", [ - {default, 5369}, - {datatype, integer} -]}. - -%% Default TCP port for outgoing connections +%% Number of tcp connections when connecting to RPC server {mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ {default, 0}, {datatype, integer}, @@ -451,6 +445,14 @@ end}. fun(X) -> X >= 0 andalso X < 256 end }. +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +{translation, "gen_rpc.tcp_client_port", fun(_, _, Conf) -> + cuttlefish:conf_get("rpc.tcp_server_port", Conf) +end}. + %%-------------------------------------------------------------------- %% Log %%-------------------------------------------------------------------- From fb643fc53f1dfaabf226b225c4f947d4de464a56 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Sat, 8 May 2021 14:43:14 +0800 Subject: [PATCH 24/58] chore(CI): add apps version check --- .github/workflows/apps_version_check.yaml | 12 ++++++++ scripts/apps-version-check.sh | 34 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/workflows/apps_version_check.yaml create mode 100755 scripts/apps-version-check.sh diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml new file mode 100644 index 000000000..808e82600 --- /dev/null +++ b/.github/workflows/apps_version_check.yaml @@ -0,0 +1,12 @@ +name: Check Apps Version + +on: [pull_request] + +jobs: + check_deps_integrity: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v1 + - name: Check apps version + run: ./scripts/apps-version-check.sh diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh new file mode 100755 index 000000000..94b3c775e --- /dev/null +++ b/scripts/apps-version-check.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1)") + +bad_app_count=0 + +while read -r app; do + if [ "$app" != "emqx" ]; then + app_path="$app" + else + app_path="." + fi + src_file="$app_path/src/$(basename "$app").app.src" + old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" + now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') + if [ "$old_app_version" = "$now_app_version" ]; then + changed="$(git diff --name-only "$latest_release"...HEAD \ + -- "$app_path/etc" \ + -- "$app_path/src" \ + -- "$app_path/priv" \ + -- "$app_path/c_src" | wc -l)" + if [ "$changed" -gt 0 ]; then + echo "$src_file needs a vsn bump" + bad_app_count=$(( bad_app_count + 1)) + fi + fi +done < <(./scripts/find-apps.sh) + +if [ $bad_app_count -gt 0 ]; then + exit 1 +else + echo "apps version check successfully" +fi From 2ab9be3e2b992112bc672c3918cb765c525afc18 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 11 May 2021 14:03:26 +0200 Subject: [PATCH 25/58] fix(emqx_ctl): ensure env variables are exported --- bin/emqx_ctl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/emqx_ctl b/bin/emqx_ctl index 73f2fc960..4c3aaa1ca 100755 --- a/bin/emqx_ctl +++ b/bin/emqx_ctl @@ -8,6 +8,9 @@ ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$ROOT_DIR"/releases/emqx_vars +export RUNNER_ROOT_DIR +export REL_VSN + # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" if [ -z "$LATEST_VM_ARGS" ]; then From 4577f6653ccd7f9b8ab47d370a5246e2e1d2b644 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Tue, 11 May 2021 14:20:14 +0200 Subject: [PATCH 26/58] fix(emqx_rule_engine): add appup --- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- .../src/emqx_rule_engine.appup.src | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_rule_engine/src/emqx_rule_engine.appup.src diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 14483b9b2..86846ccde 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src new file mode 100644 index 000000000..17cecac68 --- /dev/null +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -0,0 +1,14 @@ +%% -*-: erlang -*- +{"4.3.1", + [ {"4.3.0", + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. From 0788aa972f3ea4bf1911e1e3b80f9bb65cefa2a0 Mon Sep 17 00:00:00 2001 From: Rory-Z Date: Wed, 12 May 2021 01:19:07 +0000 Subject: [PATCH 27/58] chore(CI): fix app version check --- .github/workflows/apps_version_check.yaml | 2 +- scripts/apps-version-check.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 808e82600..4975e5c11 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -3,7 +3,7 @@ name: Check Apps Version on: [pull_request] jobs: - check_deps_integrity: + check_apps_version: runs-on: ubuntu-20.04 steps: diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 94b3c775e..49350b007 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1)") +latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1 --remotes=refs/remote/origin)") bad_app_count=0 From 72e945329325fd4cbfd592f2cd582f3af00a3141 Mon Sep 17 00:00:00 2001 From: wwhai Date: Wed, 12 May 2021 22:13:26 +0800 Subject: [PATCH 28/58] fix(mgmt): fix diylazer warning for function return (#4786) --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_management.appup.src | 12 ++++++++++++ apps/emqx_management/src/emqx_mgmt_data_backup.erl | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 apps/emqx_management/src/emqx_management.appup.src diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index b1e04c439..fe65052cf 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src new file mode 100644 index 000000000..5048e4f0f --- /dev/null +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -0,0 +1,12 @@ +%% -*-: erlang -*- +{"4.3.1", + [ {"4.3.0", + [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + ]} + ], + [ + {"4.3.0", + [ {load_module, emqx_mgmt_data_backup, brutal_purge, soft_purge, []} + ]} + ] +}. \ No newline at end of file diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index c9df698ed..1eb6f8245 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -514,7 +514,7 @@ import_modules(Modules) -> <<"enabled">> := Enabled, <<"created_at">> := CreatedAt, <<"description">> := Description}) -> - emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) + _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description}) end, Modules) end. From 52e02b251e3a84e024fffd114a8e15dbdbd5b87d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:10:49 +0800 Subject: [PATCH 29/58] fix(metrics): set the retained/delayed metrics to counter type The metrics for messages.* shows the number of times such messages are received, and it should be a counter rather than a gauge type. --- src/emqx.appup.src | 18 ++++++++++++------ src/emqx_metrics.erl | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 5b6a5a4d5..b10c14a9c 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,14 +1,20 @@ %% -*-: erlang -*- {VSN, - [ {"4.3.0", - [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} - ]}, + [ + {"4.3.0", [ + {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + {load_module, emqx_metrics, brutal_purge, soft_purge, []}, + {apply, {emqx_metrics, upgrade_retained_delayed_counter_type, []}} + ]}, {<<".*">>, []} ], [ - {"4.3.0", - [ {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []} - ]}, + {"4.3.0", [ + {load_module, emqx_logger_jsonfmt, brutal_purge, soft_purge, []}, + %% Just load the module. We don't need to change the 'messages.retained' + %% and 'messages.retained' counter type. + {load_module, emqx_metrics, brutal_purge, soft_purge, []} + ]}, {<<".*">>, []} ] }. diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 44009d4e2..3b96bfbad 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -65,6 +65,10 @@ , code_change/3 ]). +%% BACKW: v4.3.0 +-export([ upgrade_retained_delayed_counter_type/0 + ]). + -export_type([metric_idx/0]). -compile({inline, [inc/1, inc/2, dec/1, dec/2]}). @@ -145,8 +149,8 @@ {counter, 'messages.dropped.expired'}, % QoS2 Messages expired {counter, 'messages.dropped.no_subscribers'}, % Messages dropped {counter, 'messages.forward'}, % Messages forward - {gauge, 'messages.retained'}, % Messages retained - {gauge, 'messages.delayed'}, % Messages delayed + {counter, 'messages.retained'}, % Messages retained + {counter, 'messages.delayed'}, % Messages delayed {counter, 'messages.delivered'}, % Messages delivered {counter, 'messages.acked'} % Messages acked ]). @@ -195,6 +199,19 @@ start_link() -> -spec(stop() -> ok). stop() -> gen_server:stop(?SERVER). +%% BACKW: v4.3.0 +upgrade_retained_delayed_counter_type() -> + case ets:info(?TAB, name) of + ?TAB -> + [M1] = ets:lookup(?TAB, 'messages.retained'), + [M2] = ets:lookup(?TAB, 'messages.delayed'), + true = ets:insert(?TAB, M1#metric{type = counter}), + true = ets:insert(?TAB, M2#metric{type = counter}), + ok; + _ -> + ok + end. + %%-------------------------------------------------------------------- %% Metrics API %%-------------------------------------------------------------------- From d317d57f3f329b402187d82199aa78dc50efa478 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:16:21 +0800 Subject: [PATCH 30/58] fix(metric): incr the messages.retained counter --- apps/emqx_retainer/src/emqx_retainer.appup.src | 15 +++++++++++++++ apps/emqx_retainer/src/emqx_retainer.erl | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_retainer/src/emqx_retainer.appup.src diff --git a/apps/emqx_retainer/src/emqx_retainer.appup.src b/apps/emqx_retainer/src/emqx_retainer.appup.src new file mode 100644 index 000000000..a17e6ee2f --- /dev/null +++ b/apps/emqx_retainer/src/emqx_retainer.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {load_module, emqx_retainer, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {load_module, emqx_retainer, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index c1523b7a1..340e6929d 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -200,7 +200,7 @@ sort_retained(Msgs) -> store_retained(Msg = #message{topic = Topic, payload = Payload}, Env) -> case {is_table_full(Env), is_too_big(size(Payload), Env)} of {false, false} -> - ok = emqx_metrics:set('messages.retained', retained_count()), + ok = emqx_metrics:inc('messages.retained'), mnesia:dirty_write(?TAB, #retained{topic = topic2tokens(Topic), msg = Msg, expiry_time = get_expiry_time(Msg, Env)}); From 793c813d02d060186b437c47cf49ad9ff15758ff Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:18:31 +0800 Subject: [PATCH 31/58] feat(delayed): record the delayed table size by emqx_stats There are three indictors about delayed module: - 'messages.delayed': shows the number of delayed msg received - 'emqx_delayed_count': shows the delayed table size now - 'emqx_delayed_max': shows the maximum size that the delayed table has reached --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- lib-ce/emqx_modules/src/emqx_mod_delayed.erl | 39 ++++++++++++++----- lib-ce/emqx_modules/src/emqx_modules.app.src | 2 +- .../emqx_modules/src/emqx_modules.appup.src | 15 +++++++ 4 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 lib-ce/emqx_modules/src/emqx_modules.appup.src diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index feb18ac2e..c3ffb9f90 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -1,6 +1,6 @@ {application, emqx_retainer, [{description, "EMQ X Retainer"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl index 852fc4717..ac5be58b2 100644 --- a/lib-ce/emqx_modules/src/emqx_mod_delayed.erl +++ b/lib-ce/emqx_modules/src/emqx_mod_delayed.erl @@ -22,6 +22,8 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-logger_header("[Delayed]"). + %% Mnesia bootstrap -export([mnesia/1]). @@ -90,7 +92,11 @@ description() -> %% Hooks %%-------------------------------------------------------------------- -on_message_publish(Msg = #message{id = Id, topic = <<"$delayed/", Topic/binary>>, timestamp = Ts}) -> +on_message_publish(Msg = #message{ + id = Id, + topic = <<"$delayed/", Topic/binary>>, + timestamp = Ts + }) -> [Delay, Topic1] = binary:split(Topic, <<"/">>), PubAt = case binary_to_integer(Delay) of Interval when Interval < ?MAX_INTERVAL -> @@ -127,42 +133,57 @@ store(DelayedMsg) -> %%-------------------------------------------------------------------- init([]) -> - {ok, ensure_publish_timer(#{timer => undefined, publish_at => 0})}. + {ok, ensure_stats_event( + ensure_publish_timer(#{timer => undefined, publish_at => 0}))}. handle_call({store, DelayedMsg = #delayed_message{key = Key}}, _From, State) -> ok = mnesia:dirty_write(?TAB, DelayedMsg), - emqx_metrics:set('messages.delayed', delayed_count()), + emqx_metrics:inc('messages.delayed'), {reply, ok, ensure_publish_timer(Key, State)}; handle_call(Req, _From, State) -> - ?LOG(error, "[Delayed] Unexpected call: ~p", [Req]), + ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. handle_cast(Msg, State) -> - ?LOG(error, "[Delayed] Unexpected cast: ~p", [Msg]), + ?LOG(error, "Unexpected cast: ~p", [Msg]), {noreply, State}. %% Do Publish... handle_info({timeout, TRef, do_publish}, State = #{timer := TRef}) -> DeletedKeys = do_publish(mnesia:dirty_first(?TAB), os:system_time(seconds)), lists:foreach(fun(Key) -> mnesia:dirty_delete(?TAB, Key) end, DeletedKeys), - emqx_metrics:set('messages.delayed', delayed_count()), {noreply, ensure_publish_timer(State#{timer := undefined, publish_at := 0})}; +handle_info(stats, State = #{stats_fun := StatsFun}) -> + StatsFun(delayed_count()), + {noreply, State, hibernate}; + handle_info(Info, State) -> - ?LOG(error, "[Delayed] Unexpected info: ~p", [Info]), + ?LOG(error, "Unexpected info: ~p", [Info]), {noreply, State}. terminate(_Reason, #{timer := TRef}) -> emqx_misc:cancel_timer(TRef). -code_change(_OldVsn, State, _Extra) -> - {ok, State}. +code_change({down, Vsn}, State, _Extra) when Vsn =:= "4.3.0" -> + NState = maps:with([timer, publish_at], State), + {ok, NState}; + +code_change(Vsn, State, _Extra) when Vsn =:= "4.3.0" -> + NState = ensure_stats_event(State), + {ok, NState}. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- +%% Ensure the stats +ensure_stats_event(State) -> + StatsFun = emqx_stats:statsfun('delayed.count', 'delayed.max'), + {ok, StatsTimer} = timer:send_interval(timer:seconds(1), stats), + State#{stats_fun => StatsFun, stats_timer => StatsTimer}. + %% Ensure publish timer ensure_publish_timer(State) -> ensure_publish_timer(mnesia:dirty_first(?TAB), State). diff --git a/lib-ce/emqx_modules/src/emqx_modules.app.src b/lib-ce/emqx_modules/src/emqx_modules.app.src index a969fd30d..576316703 100644 --- a/lib-ce/emqx_modules/src/emqx_modules.app.src +++ b/lib-ce/emqx_modules/src/emqx_modules.app.src @@ -1,6 +1,6 @@ {application, emqx_modules, [{description, "EMQ X Module Management"}, - {vsn, "4.3.0"}, + {vsn, "4.3.1"}, {modules, []}, {applications, [kernel,stdlib]}, {mod, {emqx_modules_app, []}}, diff --git a/lib-ce/emqx_modules/src/emqx_modules.appup.src b/lib-ce/emqx_modules/src/emqx_modules.appup.src new file mode 100644 index 000000000..b44a65c17 --- /dev/null +++ b/lib-ce/emqx_modules/src/emqx_modules.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {update, emqx_mod_delayed, {advanced, []}} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {update, emqx_mod_delayed, {advanced, []}} + ]}, + {<<".*">>, []} + ] +}. From 02d9740d14ed2d08490e9ce9eeb94b64ccdce6e3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 7 May 2021 15:07:06 +0800 Subject: [PATCH 32/58] feat(lwm2m): observe specified object lists --- apps/emqx_lwm2m/etc/emqx_lwm2m.conf | 8 ++++++-- apps/emqx_lwm2m/priv/emqx_lwm2m.schema | 12 ++++++++++-- apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl | 17 ++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf index 3fec51d47..b244fa385 100644 --- a/apps/emqx_lwm2m/etc/emqx_lwm2m.conf +++ b/apps/emqx_lwm2m/etc/emqx_lwm2m.conf @@ -13,8 +13,12 @@ lwm2m.lifetime_max = 86400s # the downlink commands sent to the client will be cached. #lwm2m.qmode_time_window = 22 -# Auto send observer command to device -# on | off +# Auto send observer command to device. It can be configured as an OjbectList +# so that emqx will automatically observe the objects in this list. +# +# For examples: "/3/0,/3/0/1,/32976" +# +# Value: off | on | String #lwm2m.auto_observe = off # The topic subscribed by the lwm2m client after it is connected diff --git a/apps/emqx_lwm2m/priv/emqx_lwm2m.schema b/apps/emqx_lwm2m/priv/emqx_lwm2m.schema index b5ed778f5..bf5f144e0 100644 --- a/apps/emqx_lwm2m/priv/emqx_lwm2m.schema +++ b/apps/emqx_lwm2m/priv/emqx_lwm2m.schema @@ -26,8 +26,8 @@ ]}. {mapping, "lwm2m.auto_observe", "emqx_lwm2m.auto_observe", [ - {datatype, flag}, - {default, off} + {datatype, string}, + {default, "off"} %% BACKW: v4.3.0 ]}. {mapping, "lwm2m.lb", "emqx_lwm2m.options", [ @@ -39,6 +39,14 @@ {datatype, bytesize} ]}. +{translation, "emqx_lwm2m.auto_observe", fun(Conf) -> + case cuttlefish:conf_get("lwm2m.auto_observe", Conf, "off") of + "off" -> false; %% BACKW: v4.3.0 + "on" -> true; %% BACKW: v4.3.0 + Str -> string:tokens(Str, ", ") + end +end}. + {translation, "emqx_lwm2m.bind_udp", fun(Conf) -> Options = cuttlefish_variable:filter_by_prefix("lwm2m.bind.udp", Conf), lists:map(fun({_, Bind}) -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index bcc2267cd..55f992da6 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -275,13 +275,24 @@ do_send_to_broker(EventType, Payload, Lwm2mState) -> %% Auto Observe %%-------------------------------------------------------------------- +auto_observe_object_list(true = _Expected, Registered) -> + Registered; +auto_observe_object_list(Expected, Registered) -> + Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), + lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). + send_auto_observe(CoapPid, RegInfo) -> %% - auto observe the objects case proplists:get_value(auto_observe, lwm2m_coap_responder:options(), false) of - true -> + false -> + ?LOG(info, "Auto Observe Disabled", []); + TrueOrObjList -> + Objectlists = auto_observe_object_list( + TrueOrObjList, + maps:get(<<"objectList">>, RegInfo, []) + ), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, maps:get(<<"objectList">>, RegInfo, []), CoapPid); - _ -> ?LOG(info, "Auto Observe Disabled", []) + auto_observe(AlternatePath, Objectlists, CoapPid) end. auto_observe(AlternatePath, ObjectList, CoapPid) -> From 13257be6b62408433352da53a3b7a0a46c3906b0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 09:39:23 +0800 Subject: [PATCH 33/58] chore(appup): supply the appup instructions --- apps/emqx_lwm2m/src/emqx_lwm2m.appup.src | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m.appup.src diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src new file mode 100644 index 000000000..fb22c843d --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -0,0 +1,15 @@ +%% -*-: erlang -*- +{VSN, + [ + {"4.3.0", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ], + [ + {"4.3.0", [ + {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} + ]}, + {<<".*">>, []} + ] +}. From 7dd107a7cdade89b94701b9ffc98cb4970630ba0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 May 2021 14:53:07 +0800 Subject: [PATCH 34/58] chore(prom): update grafana templates --- .../emqx_prometheus/grafana_template/EMQ.json | 1048 +++++++++++++- .../grafana_template/EMQ_Dashboard.json | 111 +- .../grafana_template/ErlangVM.json | 1217 +++++++++++++---- 3 files changed, 2050 insertions(+), 326 deletions(-) diff --git a/apps/emqx_prometheus/grafana_template/EMQ.json b/apps/emqx_prometheus/grafana_template/EMQ.json index 41dbb0be1..137e3a5a4 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ.json +++ b/apps/emqx_prometheus/grafana_template/EMQ.json @@ -17,7 +17,7 @@ "gnetId": null, "graphTooltip": 0, "id": 2, - "iteration": 1589443996970, + "iteration": 1620639954651, "links": [], "panels": [ { @@ -26,6 +26,12 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -54,9 +60,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -66,13 +73,22 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_connections_count{instance=\"$host\"}", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Clients", + "legendFormat": "Current", "refId": "A", "step": 60 + }, + { + "exemplar": true, + "expr": "emqx_connections_max{instance=\"$host\"}", + "hide": false, + "interval": "1m", + "legendFormat": "Max", + "refId": "B" } ], "thresholds": [], @@ -122,7 +138,11 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -132,6 +152,228 @@ "y": 0 }, "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_sessions_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_sessions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Session", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_suboptions_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Subopt-Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_suboptions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Subopt-Max", + "refId": "B" + }, + { + "exemplar": true, + "expr": "emqx_subscribers_count{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Suber-Current", + "refId": "C" + }, + { + "exemplar": true, + "expr": "emqx_subscribers_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Suber-Max", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Subscribers & Suboptions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, "id": 7, "legend": { "avg": false, @@ -147,9 +389,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -159,12 +402,38 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_subscriptions_count{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "Subscriptions", + "legendFormat": "Normal-Current", "refId": "A", "step": 60 + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Normal-Max", + "refId": "B" + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_shared_count{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Shared-Current", + "refId": "C" + }, + { + "exemplar": true, + "expr": "emqx_subscriptions_shared_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Shared-Max", + "refId": "D" } ], "thresholds": [], @@ -215,13 +484,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 7 + "y": 14 }, "hiddenSeries": false, "id": 12, @@ -239,9 +514,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -251,11 +527,21 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "emqx_routes_count{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 2, - "legendFormat": "Topics", + "legendFormat": "Current", "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_routes_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" } ], "thresholds": [], @@ -305,14 +591,123 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 7 + "y": 14 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_retained_count{instance=\"$host\"}", + "interval": "", + "legendFormat": "Current", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_retained_max{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Max", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Retained Messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 20 }, "hiddenSeries": false, "id": 13, @@ -330,9 +725,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -342,6 +738,7 @@ "steppedLine": false, "targets": [ { + "exemplar": true, "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -350,6 +747,7 @@ "refId": "A" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -358,6 +756,7 @@ "refId": "B" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_retained{instance=\"$host\"}[$__interval]) or irate(emqx_messages_retained{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", @@ -366,19 +765,36 @@ "refId": "C" }, { + "exemplar": true, "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "Dropped", "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_delayed{instance=\"$host\"}[$__interval]) or irate(emqx_messages_delayed{instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delayed", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_forward{instance=\"$host\"}[$__interval]) or irate(emqx_messages_forward{instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Forward", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Messages", + "title": "Messages Overview", "tooltip": { "shared": true, "sort": 0, @@ -422,13 +838,433 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped{instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Message.Dropped", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_expired {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Message.Dropped.Expired", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]) or irate(emqx_messages_dropped_no_subscribers {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Message.Dropped.no_subscribers", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped", + "refId": "D" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_no_local{instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_no_local {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.no_local", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_too_large {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.too_large", + "refId": "F" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_qos0_msg {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.qos0_msg", + "refId": "G" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_queue_full {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.queue_full", + "refId": "H" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]) or irate(emqx_delivery_dropped_expired {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Delivery.Dropped.expired", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages/Delivery Dropped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, "w": 12, "x": 0, - "y": 14 + "y": 26 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_received{instance=\"$host\"}[$__interval]) or irate(emqx_messages_received{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Received", + "refId": "A" + }, + { + "exemplar": false, + "expr": "sum(rate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS0", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS1", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_received {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Received.QoS2", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(emqx_messages_sent{instance=\"$host\"}[$__interval]) or irate(emqx_messages_sent{instance=\"$host\"}[$__interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Sent", + "refId": "E" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos0_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS0", + "refId": "F" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos1_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS1", + "refId": "G" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]) or irate(emqx_messages_qos2_sent {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Sent.QoS2", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 32 }, "hiddenSeries": false, "id": 9, @@ -447,9 +1283,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -524,13 +1361,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 14 + "y": 32 }, "hiddenSeries": false, "id": 2, @@ -554,9 +1397,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -566,7 +1410,8 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_connect{instance=\"$host\"}[$__interval]) or irate(emqx_packets_connect{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_connect{instance=\"$host\"}[$__interval])", "format": "time_series", "hide": false, "interval": "$interval", @@ -576,7 +1421,8 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_connack_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -585,13 +1431,47 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_disconnect_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Disconnect sent", "refId": "C", "step": 60 + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_connack_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "connack_error", + "refId": "D" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_connack_auth_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "connack_auth_error", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_auth_received {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Auth", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_auth_sent {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "Auth sent", + "refId": "G" } ], "thresholds": [], @@ -642,13 +1522,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 21 + "y": 38 }, "hiddenSeries": false, "hideTimeOverride": false, @@ -671,9 +1557,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -684,8 +1571,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_received{instance=\"$host\"}[$__interval])", "format": "time_series", + "instant": false, "interval": "$interval", "intervalFactor": 1, "legendFormat": "Subscribe received", @@ -694,7 +1583,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_suback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -703,7 +1593,8 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_unsubscribe_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -712,13 +1603,38 @@ "step": 60 }, { - "expr": "sum(rate(emqx_packets_unsuback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_unsuback_send{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_unsuback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "Unsuback sent", "refId": "D", "step": 60 + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_auth_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "sub_auth_error", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_subscribe_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "sub_error", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(emqx_packets_unsubscribe_error {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "unsub_error", + "refId": "G" } ], "thresholds": [], @@ -769,13 +1685,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 21 + "y": 38 }, "hiddenSeries": false, "id": 10, @@ -794,9 +1716,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -873,13 +1796,19 @@ "dashes": false, "datasource": "Prometheus", "decimals": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 28 + "y": 44 }, "hiddenSeries": false, "hideTimeOverride": false, @@ -900,9 +1829,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -912,7 +1842,8 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_packets_publish_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_publish_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_publish_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -921,7 +1852,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_publish_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -930,7 +1862,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_puback_received{instance=\"$host\"}[$__interval]) or irate(emqx_packets_puback_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_puback_received{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 1, @@ -940,7 +1873,8 @@ "step": 15 }, { - "expr": "sum(rate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval]) or irate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_puback_sent{instance=\"$host\"}[$__interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -949,7 +1883,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubrel_received{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubrel_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubrel_received{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -958,7 +1893,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubrel_sent{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubrel_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubrel_sent{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -967,7 +1903,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubcomp_received{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubcomp_received{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubcomp_received{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -976,7 +1913,8 @@ "step": 30 }, { - "expr": "sum(rate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$interval]) or irate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$__interval]))", + "exemplar": true, + "expr": "rate(emqx_packets_pubcomp_sent{instance=\"$host\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, @@ -1029,7 +1967,7 @@ } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -1043,6 +1981,8 @@ "text": "auto", "value": "$__auto_interval_interval" }, + "description": null, + "error": null, "hide": 0, "label": "Interval", "name": "interval", @@ -1096,19 +2036,24 @@ { "allValue": null, "current": { - "text": "emqx~127.0.0.1", - "value": "emqx~127.0.0.1" + "selected": false, + "text": "128.160.171.92:8081", + "value": "128.160.171.92:8081" }, "datasource": "Prometheus", "definition": "label_values(emqx_connections_count, instance)", + "description": null, + "error": null, "hide": 0, "includeAll": false, - "index": -1, "label": "Host", "multi": false, "name": "host", "options": [], - "query": "label_values(emqx_connections_count, instance)", + "query": { + "query": "label_values(emqx_connections_count, instance)", + "refId": "Prometheus-host-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1153,8 +2098,5 @@ "timezone": "browser", "title": "EMQ", "uid": "tjRlQw6Zk", - "variables": { - "list": [] - }, - "version": 6 + "version": 29 } \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json index 661f4d739..0b0e2036b 100644 --- a/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json +++ b/apps/emqx_prometheus/grafana_template/EMQ_Dashboard.json @@ -16,7 +16,7 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 1, + "id": 3, "links": [], "panels": [ { @@ -25,10 +25,16 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 10, "w": 24, "x": 0, "y": 0 @@ -53,9 +59,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -65,23 +72,33 @@ "steppedLine": false, "targets": [ { - "expr": "sum(emqx_client_connected{instance=~\".*\",job=\"emqx\"})", + "exemplar": true, + "expr": "sum(emqx_connections_count{instance=~\".*\",job=\"emqx\"})", + "hide": false, + "interval": "", + "legendFormat": "Clients", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(emqx_client_connected{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "instant": false, "interval": "1m", "intervalFactor": 1, - "legendFormat": "connected", - "refId": "A", + "legendFormat": "connected rate", + "refId": "B", "step": 60 }, { - "expr": "sum(emqx_client_disconnected{instance=~\".*\",job=\"emqx\"})", + "exemplar": true, + "expr": "sum(rate(emqx_client_disconnected{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "instant": false, "interval": "1m", "intervalFactor": 1, - "legendFormat": "disconnected", - "refId": "B", + "legendFormat": "disconnected rate", + "refId": "C", "step": 60 } ], @@ -136,13 +153,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 24, "x": 0, - "y": 7 + "y": 10 }, "hiddenSeries": false, "id": 11, @@ -161,9 +184,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -173,20 +197,22 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_messages_received{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_messages_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Messages received", + "legendFormat": "received", "refId": "A", "step": 120 }, { - "expr": "sum(rate(emqx_messages_sent{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_messages_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", "interval": "1m", "intervalFactor": 1, - "legendFormat": "Messages sent", + "legendFormat": "sent", "refId": "B", "step": 120 } @@ -240,13 +266,19 @@ "dashes": false, "datasource": "Prometheus", "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 14 + "y": 18 }, "hiddenSeries": false, "id": 13, @@ -266,9 +298,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -337,13 +370,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 21 + "y": 25 }, "hiddenSeries": false, "id": 8, @@ -362,9 +401,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -374,7 +414,8 @@ "steppedLine": false, "targets": [ { - "expr": "emqx_subscribers_count{instance=~\".*\",job=\"emqx\"}", + "exemplar": true, + "expr": "emqx_subscriptions_count{instance=~\".*\",job=\"emqx\"}", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -443,13 +484,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 28 + "y": 32 }, "hiddenSeries": false, "id": 9, @@ -468,9 +515,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -480,18 +528,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(emqx_bytes_received{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_bytes_received{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "", "intervalFactor": 1, "legendFormat": "Bytes received", "refId": "A", "step": 60 }, { - "expr": "sum(rate(emqx_bytes_sent{instance=~\".*\",job=\"emqx\"}[5m]))", + "exemplar": true, + "expr": "sum(rate(emqx_bytes_sent{instance=~\".*\",job=\"emqx\"}[$__interval]))", "format": "time_series", - "interval": "1m", + "interval": "", "intervalFactor": 1, "legendFormat": "Bytes sent", "refId": "B", @@ -541,7 +591,7 @@ } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -579,8 +629,5 @@ "timezone": "browser", "title": "EMQ Dashboard", "uid": "5sreUw6Wz", - "variables": { - "list": [] - }, - "version": 6 + "version": 11 } \ No newline at end of file diff --git a/apps/emqx_prometheus/grafana_template/ErlangVM.json b/apps/emqx_prometheus/grafana_template/ErlangVM.json index 1e9321f84..556d815b0 100644 --- a/apps/emqx_prometheus/grafana_template/ErlangVM.json +++ b/apps/emqx_prometheus/grafana_template/ErlangVM.json @@ -16,8 +16,8 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 3, - "iteration": 1589444007168, + "id": 4, + "iteration": 1620639927619, "links": [], "panels": [ { @@ -25,17 +25,22 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", - "fill": 0, + "datasource": null, + "description": "The CPU usage for the hosted machine", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 8, "w": 12, "x": 0, "y": 0 }, "hiddenSeries": false, - "id": 1, + "id": 14, "legend": { "avg": false, "current": false, @@ -47,13 +52,13 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pointradius": 5, + "pluginVersion": "7.5.5", + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -62,27 +67,26 @@ "steppedLine": false, "targets": [ { - "expr": "erlang_vm_process_count{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Process count", - "refId": "A", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_cpu_use {instance=\"$host\"}", + "interval": "", + "legendFormat": "used", + "refId": "A" }, { - "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Run queue length", - "refId": "B", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_cpu_idle {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "idle", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Process", + "title": "CPU", "tooltip": { "shared": true, "sort": 0, @@ -102,7 +106,205 @@ "label": null, "logBase": 1, "max": null, - "min": "0", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "The memory usage for the hosted machine", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "emqx_vm_total_memory{instance=\"$host\"}", + "interval": "", + "legendFormat": "total", + "refId": "A" + }, + { + "exemplar": true, + "expr": "emqx_vm_used_memory{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "used", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "The total amount of memory currently allocated.\nThis is the same as the sum of the memory size for processes and system.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_memory_bytes_total {instance=\"$host\"}", + "interval": "", + "legendFormat": "{{kind}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VM Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, "show": true }, { @@ -125,203 +327,20 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "The total amount of memory currently allocated for the Erlang processes.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "erlang_vm_threads{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Threads", - "refId": "A", - "step": 60 - }, - { - "expr": "erlang_vm_thread_pool_size{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Thread pool size", - "refId": "B", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Threads", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "erlang_vm_memory_system_bytes_total{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{usage}}", - "refId": "A", - "step": 60 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory system", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 7 + "y": 8 }, "hiddenSeries": false, "id": 6, @@ -339,9 +358,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -405,16 +425,23 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "description": "The total amount of memory currently allocated for the emulator that is not directly related to any Erlang process.\nMemory presented as processes is not included in this memory.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 14 + "y": 15 }, "hiddenSeries": false, - "id": 4, + "id": 2, "legend": { "avg": false, "current": false, @@ -429,9 +456,568 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "erlang_vm_memory_system_bytes_total{instance=\"$host\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{usage}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory system", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_logical_processors {instance=\"$host\"}", + "interval": "", + "legendFormat": "logical-processors", + "refId": "A" + }, + { + "exemplar": true, + "expr": "erlang_vm_logical_processors_available {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "logical-processors-avai", + "refId": "B" + }, + { + "exemplar": true, + "expr": "erlang_vm_logical_processors_online {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "logical-processors-online", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_vm_schedulers {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "schedulers", + "refId": "D" + }, + { + "exemplar": true, + "expr": "erlang_vm_schedulers_online {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "schedulers-online", + "refId": "E" + }, + { + "exemplar": true, + "expr": "erlang_vm_threads {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_threads", + "refId": "F" + }, + { + "exemplar": true, + "expr": "erlang_vm_thread_pool_size {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_thread_pool_size", + "refId": "G" + }, + { + "exemplar": true, + "expr": "emqx_vm_run_queue {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "vm_run_queue", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Processor & Scheduler & Thread", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "erlang_vm_ets_limit {instance=\"$host\"}", + "interval": "", + "legendFormat": "ETS Max", + "refId": "A" + }, + { + "exemplar": true, + "expr": "erlang_vm_ets_tables {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "ETS Count", + "refId": "B" + }, + { + "exemplar": true, + "expr": "erlang_vm_dets_tables {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "DETS Count", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_vm_process_limit {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Process Max", + "refId": "D" + }, + { + "exemplar": true, + "expr": "erlang_vm_process_count {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Process Count", + "refId": "E" + }, + { + "exemplar": true, + "expr": "erlang_vm_port_limit {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Port Max", + "refId": "F" + }, + { + "exemplar": true, + "expr": "erlang_vm_port_count {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Port Count", + "refId": "G" + }, + { + "exemplar": true, + "expr": "erlang_vm_smp_support {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "smp_support", + "refId": "H" + }, + { + "exemplar": true, + "expr": "erlang_vm_time_correction {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "time_correction", + "refId": "I" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System Limit", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_context_switches{instance=\"$host\"}[$__interval])", + "interval": "", + "legendFormat": "context_switches", + "refId": "A" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_garbage_collection_number_of_gcs {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "garbage_collection_number_of_gcs", + "refId": "B" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_garbage_collection_bytes_reclaimed {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "garbage_collection_bytes_reclaimed", + "refId": "D" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_bytes_received_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "bytes_received_total", + "refId": "C" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_bytes_output_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "bytes_output_total", + "refId": "E" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_reductions_total {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "reductions", + "refId": "F" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_runtime_milliseconds {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "runtime_milliseconds", + "refId": "G" + }, + { + "exemplar": true, + "expr": "rate(erlang_vm_statistics_wallclock_time_milliseconds {instance=\"$host\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "wallclock_time_milliseconds", + "refId": "H" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VM Statistics", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -441,35 +1027,35 @@ "steppedLine": false, "targets": [ { - "expr": "erlang_vm_ets_tables{instance=\"$host\"}", + "expr": "erlang_vm_process_count{instance=\"$host\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "ETS tables", + "legendFormat": "Process count", "refId": "A", "step": 60 }, { - "expr": "erlang_vm_ets_limit{instance=\"$host\"}", + "expr": "erlang_vm_statistics_run_queues_length_total{instance=\"$host\"}", "format": "time_series", "intervalFactor": 2, - "legendFormat": "ETS limit", + "legendFormat": "Run queue length", "refId": "B", "step": 60 }, { - "expr": "erlang_vm_dets_tables{instance=\"$host\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "DETS tables", - "refId": "C", - "step": 60 + "exemplar": true, + "expr": "emqx_vm_process_messages_in_queues {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "messages_in_queue", + "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "ETS", + "title": "Process", "tooltip": { "shared": true, "sort": 0, @@ -512,13 +1098,19 @@ "dashLength": 10, "dashes": false, "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 14 + "y": 29 }, "hiddenSeries": false, "id": 5, @@ -536,9 +1128,10 @@ "links": [], "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.5", "pointradius": 5, "points": false, "renderer": "flot", @@ -556,12 +1149,30 @@ "step": 30 }, { + "exemplar": true, "expr": "erlang_mnesia_held_locks{instance=\"$host\"}", "format": "time_series", + "interval": "", "intervalFactor": 2, "legendFormat": "Held locks", "refId": "B", "step": 30 + }, + { + "exemplar": true, + "expr": "erlang_mnesia_transaction_coordinators{instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Transaction coordinators", + "refId": "C" + }, + { + "exemplar": true, + "expr": "erlang_mnesia_transaction_participants {instance=\"$host\"}", + "hide": false, + "interval": "", + "legendFormat": "Transaction participants", + "refId": "D" } ], "thresholds": [], @@ -605,10 +1216,129 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_committed_transactions {instance=\"$host\"}[$__interval]))", + "interval": "", + "legendFormat": "Committed", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_failed_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Failed", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_restarted_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Restart", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum(rate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]) or irate(erlang_mnesia_logged_transactions {instance=\"$host\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "Logged", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mnesia transaction", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -618,10 +1348,12 @@ "auto_count": 200, "auto_min": "1s", "current": { - "selected": false, + "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, + "description": null, + "error": null, "hide": 0, "label": "Interval", "name": "interval", @@ -668,6 +1400,7 @@ } ], "query": "1s,5s,1m,5m,1h,6h,1d", + "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "interval" @@ -675,19 +1408,24 @@ { "allValue": null, "current": { - "text": "emqx~127.0.0.1", - "value": "emqx~127.0.0.1" + "selected": false, + "text": "128.160.171.92:8081", + "value": "128.160.171.92:8081" }, "datasource": "Prometheus", "definition": "", + "description": null, + "error": null, "hide": 0, "includeAll": false, - "index": -1, "label": "Host", "multi": false, "name": "host", "options": [], - "query": "label_values(erlang_vm_process_count, instance)", + "query": { + "query": "label_values(erlang_vm_process_count, instance)", + "refId": "Prometheus-host-Variable-Query" + }, "refresh": 1, "regex": "", "skipUrlSync": false, @@ -732,8 +1470,5 @@ "timezone": "browser", "title": "ErlangVM", "uid": "stprQQ6Zk", - "variables": { - "list": [] - }, - "version": 1 + "version": 13 } \ No newline at end of file From 3e3819905d3837f647aba134dd750e3027969c2c Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 12 May 2021 22:09:57 +0800 Subject: [PATCH 35/58] fix(import): fix import bridge mqtt test cases --- .../test/emqx_bridge_mqtt_data_export_import_SUITE.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl index 7cf7d19cf..aca60a687 100644 --- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl +++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl @@ -177,7 +177,6 @@ handle_config(Config, ee435, Id) -> remove_resources() -> timer:sleep(500), lists:foreach(fun(#resource{id = Id}) -> - emqx_rule_registry:remove_resource(Id), - emqx_rule_registry:remove_resource_params(Id) + emqx_rule_engine:delete_resource(Id) end, emqx_rule_registry:get_resources()), timer:sleep(500). \ No newline at end of file From 35c38e7d211487a0a232db1449cfe7a07b53e5f3 Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 13 May 2021 09:27:18 +0800 Subject: [PATCH 36/58] fix(lwm2m): fix check emqx-lwm2m version --- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index a576a4149..19fabc526 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, From db4a76fe752979cb68a908e913b279324042b968 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 13 May 2021 13:51:17 +0800 Subject: [PATCH 37/58] chore(CI): relup test support any commit --- .ci/fvt_tests/relup.lux | 23 ++++++----------------- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/run_fvt_tests.yaml | 14 ++++++++------ 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index c0c2b7592..b75e0fa94 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -21,20 +21,15 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o emqx-ubuntu20.04-$old_vsn-x86_64.zip + !unzip -q -o emqx-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip ?SH-PROMPT !cd emqx !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X Broker $old_vsn is started successfully! - - !./bin/emqx_ctl status - """? - Node 'emqx@127.0.0.1' is started - emqx $old_vsn is running - """ + ?EMQ X .* is started successfully! + ?SH-PROMPT [shell emqx2] !cd $PACKAGE_PATH @@ -45,13 +40,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X Broker $old_vsn is started successfully! - - !./bin/emqx_ctl status - """? - Node 'emqx2@127.0.0.1' is started - emqx $old_vsn is running - """ + ?EMQ X (.*) is started successfully! ?SH-PROMPT !./bin/emqx_ctl cluster join emqx@127.0.0.1 @@ -86,7 +75,7 @@ ???sent [shell emqx] - !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" @@ -101,7 +90,7 @@ ?SH-PROMPT [shell emqx2] - !cp -f ../emqx-ubuntu20.04-$VSN-x86_64.zip releases/ + !cp -f ../emqx-ubuntu20.04-$VSN-amd64.zip releases/ !./bin/emqx install $VSN ?SH-PROMPT !./bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index fde6795b1..70419bcc1 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -256,9 +256,9 @@ jobs: if [ $PROFILE = "emqx" ];then broker="emqx-ce"; else broker="$PROFILE"; fi if [ $PROFILE = "emqx-ee" ];then edition='enterprise'; else edition='opensource'; fi - vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" + vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx-ee" ]; then + if [ $PROFILE = "emqx-ee" ]; then old_vsns=($(git tag -l "e$pre_vsn.[0-9]" | sed "s/e$vsn//")) else old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//")) @@ -268,8 +268,8 @@ jobs: cd _upgrade_base for tag in ${old_vsns[@]};do if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 fi done diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8f65d022e..eea7acde5 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -214,24 +214,26 @@ jobs: broker="emqx-ee" edition='enterprise' fi + echo "BROKER=$broker" >> $GITHUB_ENV - vsn="$(grep -E "define.+EMQX_RELEASE.+${edition}" include/emqx_release.hrl | cut -d '"' -f2)" + vsn="$(./pkg-vsn.sh)" echo "VSN=$vsn" >> $GITHUB_ENV pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx" ]; then + if [ $PROFILE = "emqx" ]; then old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" else - old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" + old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" fi echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV - name: download emqx run: | set -e -x -u - cd emqx + mkdir -p emqx/_upgrade_base + cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-x86_64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip done - name: build emqx run: make -C emqx ${PROFILE}-zip @@ -251,7 +253,7 @@ jobs: if [ -n "$OLD_VSNS" ]; then mkdir -p packages cp emqx/_packages/emqx/*.zip packages - cp emqx/*.zip packages + cp emqx/_upgrade_base/*.zip packages lux -v \ --timeout 600000 \ --var PACKAGE_PATH=$(pwd)/packages \ From c57e51af5b55cbc32511a8d039dea5182258006d Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 13 May 2021 06:58:08 +0200 Subject: [PATCH 38/58] fix(nodetool): add path from RELEASES file --- bin/nodetool | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/bin/nodetool b/bin/nodetool index 5ab5ec69a..431121148 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -296,11 +296,30 @@ add_libs_dir() -> os:getenv("REL_VSN"), "emqx.rel" ]), - {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} = file:consult(RelFile), - lists:foreach( - fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); - ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) - end, Libs). + case file:consult(RelFile) of + {ok, [{release, {_, _RelVsn}, {erts, _ErtsVsn}, Libs}]} -> + lists:foreach( + fun({Name, Vsn}) -> add_lib_dir(RootDir, Name, Vsn); + ({Name, Vsn, _}) -> add_lib_dir(RootDir, Name, Vsn) + end, Libs); + {error, enoent} -> + %% rel file is deleted by release handler + add_libs_dir2(RootDir) + end. + +add_libs_dir2(RootDir) -> + RelFile = filename:join([RootDir, "releases", "RELEASES"]), + case file:consult(RelFile) of + {ok, [[Release]]} -> + {release, _Name, _AppVsn, _ErtsVsn, Libs, _State} = Release, + lists:foreach( + fun({Name, Vsn, _}) -> + add_lib_dir(RootDir, Name, Vsn) + end, Libs); + {error, Reason} -> + %% rel file was been deleted by release handler + error({failed_to_read_RELEASES_file, RelFile, Reason}) + end. add_lib_dir(RootDir, Name, Vsn) -> LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), From 0a010237a0e5147870db730b7a64f9f57a06700b Mon Sep 17 00:00:00 2001 From: z8674558 Date: Fri, 14 May 2021 04:44:17 +0900 Subject: [PATCH 39/58] chore(emqx_release): 4.3.0 -> 5.0-pre --- include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index cc5064c69..e69b07558 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.0"}). +-define(EMQX_RELEASE, {opensource, "5.0-pre"}). -else. From 2a31c43e0de5589aa25e27a0bd3df079fe32aa94 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 May 2021 21:43:54 +0800 Subject: [PATCH 40/58] feat(emqx_resource): add behaviour emqx_resource --- apps/emqx_resource/LICENSE | 191 ++++++++++++ apps/emqx_resource/Makefile | 43 +++ apps/emqx_resource/README.md | 53 ++++ apps/emqx_resource/demo.sh | 6 + apps/emqx_resource/elvis.config | 14 + apps/emqx_resource/examples/demo.erl | 13 + apps/emqx_resource/examples/demo.md | 147 +++++++++ apps/emqx_resource/examples/log_tracer.conf | 11 + apps/emqx_resource/examples/log_tracer.erl | 45 +++ .../examples/log_tracer_schema.erl | 45 +++ apps/emqx_resource/include/emqx_resource.hrl | 19 ++ .../include/emqx_resource_behaviour.hrl | 3 + .../include/emqx_resource_utils.hrl | 38 +++ apps/emqx_resource/rebar.config | 14 + apps/emqx_resource/scripts/elvis-check.sh | 17 + apps/emqx_resource/src/emqx_resource.app.src | 17 + apps/emqx_resource/src/emqx_resource.erl | 274 ++++++++++++++++ apps/emqx_resource/src/emqx_resource_api.erl | 64 ++++ apps/emqx_resource/src/emqx_resource_app.erl | 31 ++ .../src/emqx_resource_instance.erl | 294 ++++++++++++++++++ apps/emqx_resource/src/emqx_resource_sup.erl | 48 +++ .../src/emqx_resource_transform.erl | 99 ++++++ .../src/emqx_resource_uitils.erl | 1 + .../src/emqx_resource_validator.erl | 63 ++++ 24 files changed, 1550 insertions(+) create mode 100644 apps/emqx_resource/LICENSE create mode 100644 apps/emqx_resource/Makefile create mode 100644 apps/emqx_resource/README.md create mode 100755 apps/emqx_resource/demo.sh create mode 100644 apps/emqx_resource/elvis.config create mode 100644 apps/emqx_resource/examples/demo.erl create mode 100644 apps/emqx_resource/examples/demo.md create mode 100644 apps/emqx_resource/examples/log_tracer.conf create mode 100644 apps/emqx_resource/examples/log_tracer.erl create mode 100644 apps/emqx_resource/examples/log_tracer_schema.erl create mode 100644 apps/emqx_resource/include/emqx_resource.hrl create mode 100644 apps/emqx_resource/include/emqx_resource_behaviour.hrl create mode 100644 apps/emqx_resource/include/emqx_resource_utils.hrl create mode 100644 apps/emqx_resource/rebar.config create mode 100755 apps/emqx_resource/scripts/elvis-check.sh create mode 100644 apps/emqx_resource/src/emqx_resource.app.src create mode 100644 apps/emqx_resource/src/emqx_resource.erl create mode 100644 apps/emqx_resource/src/emqx_resource_api.erl create mode 100644 apps/emqx_resource/src/emqx_resource_app.erl create mode 100644 apps/emqx_resource/src/emqx_resource_instance.erl create mode 100644 apps/emqx_resource/src/emqx_resource_sup.erl create mode 100644 apps/emqx_resource/src/emqx_resource_transform.erl create mode 100644 apps/emqx_resource/src/emqx_resource_uitils.erl create mode 100644 apps/emqx_resource/src/emqx_resource_validator.erl diff --git a/apps/emqx_resource/LICENSE b/apps/emqx_resource/LICENSE new file mode 100644 index 000000000..2f97fdd03 --- /dev/null +++ b/apps/emqx_resource/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2021, Shawn <506895667@qq.com>. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_resource/Makefile b/apps/emqx_resource/Makefile new file mode 100644 index 000000000..596b9b2a1 --- /dev/null +++ b/apps/emqx_resource/Makefile @@ -0,0 +1,43 @@ +REBAR := rebar3 + +.PHONY: all +all: es + +.PHONY: compile +compile: + $(REBAR) compile + +.PHONY: clean +clean: distclean + +.PHONY: distclean +distclean: + @rm -rf _build erl_crash.dump rebar3.crashdump + +.PHONY: xref +xref: + $(REBAR) xref + +.PHONY: eunit +eunit: compile + $(REBAR) eunit -v -c + $(REBAR) cover + +.PHONY: ct +ct: compile + $(REBAR) as test ct -v + +cover: + $(REBAR) cover + +.PHONY: dialyzer +dialyzer: + $(REBAR) dialyzer + +.PHONY: es +es: compile + $(REBAR) escriptize + +.PHONY: elvis +elvis: + ./scripts/elvis-check.sh diff --git a/apps/emqx_resource/README.md b/apps/emqx_resource/README.md new file mode 100644 index 000000000..9e91b633c --- /dev/null +++ b/apps/emqx_resource/README.md @@ -0,0 +1,53 @@ +# emqx_resource + +The `emqx_resource` is an application that manages configuration specs and runtime states +for components that need to be configured and manipulated from the emqx-dashboard. + +It is intended to be used by resources, actions, acl, auth, backend_logics and more. + +It reads the configuration spec from *.spec (in HOCON format) and provide APIs for +creating, updating and destroying resource instances among all nodes in the cluster. + +It handles the problem like storing the configs and runtime states for both resource +and resource instances, and how porting them between different emqx_resource versions. + +It may maintain the config and data in JSON or HOCON files in data/ dir. + +After restarting the emqx_resource, it re-creates all the resource instances. + +There can be foreign references between resource instances via resource-id. +So they may find each other via this Id. + +## Try it out + + $ ./demo.sh + Eshell V11.1.8 (abort with ^G) + 1> == the demo log tracer <<"log_tracer_clientid_shawn">> started. + config: #{<<"config">> => + #{<<"bulk">> => <<"10KB">>,<<"cache_log_dir">> => <<"/tmp">>, + <<"condition">> => #{<<"clientid">> => <<"abc">>}, + <<"level">> => <<"debug">>}, + <<"id">> => <<"log_tracer_clientid_shawn">>, + <<"resource_type">> => <<"log_tracer">>} + 1> emqx_resource_instance:health_check(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> is working well + state: #{health_checked => 1,logger_handler_id => abc} + ok + + 2> emqx_resource_instance:health_check(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> is working well + state: #{health_checked => 2,logger_handler_id => abc} + ok + + 3> emqx_resource_instance:query(<<"log_tracer_clientid_shawn">>, get_log). + == the demo log tracer <<"log_tracer_clientid_shawn">> received request: get_log + state: #{health_checked => 2,logger_handler_id => abc} + "this is a demo log messages..." + + 4> emqx_resource_instance:remove(<<"log_tracer_clientid_shawn">>). + == the demo log tracer <<"log_tracer_clientid_shawn">> stopped. + state: #{health_checked => 0,logger_handler_id => abc} + ok + + 5> emqx_resource_instance:query(<<"log_tracer_clientid_shawn">>, get_log). + ** exception error: {get_instance,{<<"log_tracer_clientid_shawn">>,not_found}} diff --git a/apps/emqx_resource/demo.sh b/apps/emqx_resource/demo.sh new file mode 100755 index 000000000..19cbab809 --- /dev/null +++ b/apps/emqx_resource/demo.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e + +rebar3 compile + +erl -sname abc -pa _build/default/lib/*/ebin _build/default/lib/emqx_resource/examples -s demo diff --git a/apps/emqx_resource/elvis.config b/apps/emqx_resource/elvis.config new file mode 100644 index 000000000..59aa13fbe --- /dev/null +++ b/apps/emqx_resource/elvis.config @@ -0,0 +1,14 @@ +[{elvis, [{config, [ + +#{dirs => ["src"], + filter => "*.erl", + %ignore => [], + ruleset => erl_files, + rules => [{elvis_style, operator_spaces, #{ + rules => [{right, ","}, + {right, "|"}, + {left, "|"}, + {right, "||"}, + {left, "||"}]}}, + {elvis_style, god_modules, #{limit => 100}}]} +]}]}]. diff --git a/apps/emqx_resource/examples/demo.erl b/apps/emqx_resource/examples/demo.erl new file mode 100644 index 000000000..171a80b61 --- /dev/null +++ b/apps/emqx_resource/examples/demo.erl @@ -0,0 +1,13 @@ +-module(demo). + +-export([start/0]). + +start() -> + code:load_file(log_tracer), + code:load_file(log_tracer_schema), + {ok, _} = application:ensure_all_started(minirest), + {ok, _} = application:ensure_all_started(emqx_resource), + emqx_resource:load_instances("./_build/default/lib/emqx_resource/examples"), + Handlers = [{"/", minirest:handler(#{modules => [log_tracer]})}], + Dispatch = [{"/[...]", minirest, Handlers}], + minirest:start_http(?MODULE, #{socket_opts => [inet, {port, 9900}]}, Dispatch). diff --git a/apps/emqx_resource/examples/demo.md b/apps/emqx_resource/examples/demo.md new file mode 100644 index 000000000..d7ff7f059 --- /dev/null +++ b/apps/emqx_resource/examples/demo.md @@ -0,0 +1,147 @@ +--- +theme: gaia +color: #000 +colorSecondary: #333 +backgroundColor: #fff +backgroundImage: url('https://marp.app/assets/hero-background.jpg') +paginate: true +marp: true +--- + + + +# EMQX Resource + +--- + +## What is it for + +The [emqx_resource](https://github.com/terry-xiaoyu/emqx_resource) for managing configurations and runtime states for dashboard components . + +![bg right](https://docs.emqx.cn/assets/img/rule_action_1@2x.73766093.png) + +--- + + + +# The Demo + +The little log tracer + +--- + +- The hocon schema file (log_tracer_schema.erl): + +https://github.com/terry-xiaoyu/emqx_resource/blob/main/examples/log_tracer_schema.erl + +- The callback file (log_tracer.erl): + +https://github.com/terry-xiaoyu/emqx_resource/blob/main/examples/log_tracer.erl + +--- + +Start the demo log tracer + +``` +./demo.sh +``` + +Load instance from config files (auto loaded) + +``` +## This will load all of the "*.conf" file under that directory: + +emqx_resource:load_instances("./_build/default/lib/emqx_resource/examples"). +``` + +The config file is validated against the schema (`*_schema.erl`) before loaded. + +--- + +# List Types and Instances + +- To list all the available resource types: + +``` +emqx_resource:list_types(). +emqx_resource:list_instances(). +``` + +- And there's `*_verbose` versions for these `list_*` APIs: + +``` +emqx_resource:list_types_verbose(). +emqx_resource:list_instances_verbose(). +``` + +--- +# Instance management + +- To get a resource types and instances: + +``` +emqx_resource:get_type(log_tracer). +emqx_resource:get_instance("log_tracer_clientid_shawn"). +``` + +- To create a resource instances: + +``` +emqx_resource:create("log_tracer2", log_tracer, +#{bulk => <<"1KB">>,cache_log_dir => <<"/tmp">>, + cache_logs_in => <<"memory">>,chars_limit => 1024, + condition => #{<<"app">> => <<"emqx">>}, + enable_cache => true,level => debug}). +``` + +--- + +- To update a resource: + +``` +emqx_resource:update("log_tracer2", log_tracer, #{bulk => <<"100KB">>}, []). +``` + +- To delete a resource: + +``` +emqx_resource:remove("log_tracer2"). +``` + +--- + + + +# HTTP APIs Demo + +--- + +# Get a log tracer + +To list current log tracers: + +``` +curl -s -XGET 'http://localhost:9900/log_tracer' | jq . +``` + +--- + +## Update or Create + +To update an existing log tracer or create a new one: + +``` +INST='{ + "resource_type": "log_tracer", + "config": { + "condition": { + "app": "emqx" + }, + "level": "debug", + "cache_log_dir": "/tmp", + "bulk": "10KB", + "chars_limit": 1024 + } +}' +curl -sv -XPUT 'http://localhost:9900/log_tracer/log_tracer2' -d $INST | jq . +``` diff --git a/apps/emqx_resource/examples/log_tracer.conf b/apps/emqx_resource/examples/log_tracer.conf new file mode 100644 index 000000000..7b438ec1f --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer.conf @@ -0,0 +1,11 @@ +{ + "id": "log_tracer_clientid_shawn" + "resource_type": "log_tracer" + "config": { + "condition": {"app": "emqx"} + "level": "debug" + "cache_log_dir": "/tmp" + "bulk": "10KB" + "chars_limit": 1024 + } +} \ No newline at end of file diff --git a/apps/emqx_resource/examples/log_tracer.erl b/apps/emqx_resource/examples/log_tracer.erl new file mode 100644 index 000000000..99fc5bd3b --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer.erl @@ -0,0 +1,45 @@ +-module(log_tracer). + +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +-emqx_resource_api_path("/log_tracer"). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + , on_api_reply_format/1 + , on_config_merge/3 + ]). + +%% callbacks for emqx_resource config schema +-export([fields/1]). + +fields(ConfPath) -> + log_tracer_schema:fields(ConfPath). + +on_start(InstId, Config) -> + io:format("== the demo log tracer ~p started.~nconfig: ~p~n", [InstId, Config]), + {ok, #{logger_handler_id => abc, health_checked => 0}}. + +on_stop(InstId, State) -> + io:format("== the demo log tracer ~p stopped.~nstate: ~p~n", [InstId, State]), + ok. + +on_query(InstId, Request, AfterQuery, State) -> + io:format("== the demo log tracer ~p received request: ~p~nstate: ~p~n", + [InstId, Request, State]), + emqx_resource:query_success(AfterQuery), + "this is a demo log messages...". + +on_health_check(InstId, State = #{health_checked := Checked}) -> + NState = State#{health_checked => Checked + 1}, + io:format("== the demo log tracer ~p is working well~nstate: ~p~n", [InstId, NState]), + {ok, NState}. + +on_api_reply_format(#{id := Id, status := Status, state := #{health_checked := NChecked}}) -> + #{id => Id, status => Status, checked_count => NChecked}. + +on_config_merge(OldConfig, NewConfig, _Params) -> + maps:merge(OldConfig, NewConfig). diff --git a/apps/emqx_resource/examples/log_tracer_schema.erl b/apps/emqx_resource/examples/log_tracer_schema.erl new file mode 100644 index 000000000..2b49aab7f --- /dev/null +++ b/apps/emqx_resource/examples/log_tracer_schema.erl @@ -0,0 +1,45 @@ +-module(log_tracer_schema). + +-include_lib("typerefl/include/types.hrl"). + +-export([fields/1]). + +-reflect_type([t_level/0, t_cache_logs_in/0]). + +-type t_level() :: debug | info | notice | warning | error | critical | alert | emergency. + +-type t_cache_logs_in() :: memory | file. + +fields("config") -> + [ {condition, fun condition/1} + , {level, fun level/1} + , {enable_cache, fun enable_cache/1} + , {cache_logs_in, fun cache_logs_in/1} + , {cache_log_dir, fun cache_log_dir/1} + , {bulk, fun bulk/1} + ]; +fields(_) -> []. + +condition(mapping) -> "config.condition"; +condition(type) -> map(); +condition(_) -> undefined. + +level(mapping) -> "config.level"; +level(type) -> t_level(); +level(_) -> undefined. + +enable_cache(mapping) -> "config.enable_cache"; +enable_cache(type) -> boolean(); +enable_cache(_) -> undefined. + +cache_logs_in(mapping) -> "config.cache_logs_in"; +cache_logs_in(type) -> t_cache_logs_in(); +cache_logs_in(_) -> undefined. + +cache_log_dir(mapping) -> "config.cache_log_dir"; +cache_log_dir(type) -> typerefl:regexp_string("^(.*)$"); +cache_log_dir(_) -> undefined. + +bulk(mapping) -> "config.bulk"; +bulk(type) -> typerefl:regexp_string("^[. 0-9]+(B|KB|MB|GB)$"); +bulk(_) -> undefined. diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl new file mode 100644 index 000000000..429b83c45 --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -0,0 +1,19 @@ +-type resource_type() :: module(). +-type instance_id() :: binary(). +-type resource_config() :: jsx:json_term(). +-type resource_spec() :: map(). +-type resource_state() :: term(). +-type resource_data() :: #{ + id => instance_id(), + mod => module(), + config => resource_config(), + state => resource_state(), + status => started | stopped +}. + +-type after_query() :: {OnSuccess :: after_query_fun(), OnFailed :: after_query_fun()} | + undefined. + +%% the `after_query_fun()` is mainly for callbacks that increment counters or do some fallback +%% actions upon query failure +-type after_query_fun() :: {fun((...) -> ok), Args :: [term()]}. diff --git a/apps/emqx_resource/include/emqx_resource_behaviour.hrl b/apps/emqx_resource/include/emqx_resource_behaviour.hrl new file mode 100644 index 000000000..bf0f1cd2e --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource_behaviour.hrl @@ -0,0 +1,3 @@ +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-behaviour(emqx_resource). +-compile({parse_transform, emqx_resource_transform}). diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl new file mode 100644 index 000000000..dd1517428 --- /dev/null +++ b/apps/emqx_resource/include/emqx_resource_utils.hrl @@ -0,0 +1,38 @@ +-define(CLUSTER_CALL(Func, Args), ?CLUSTER_CALL(Func, Args, ok)). + +-define(CLUSTER_CALL(Func, Args, ResParttern), +%% ekka_mnesia:running_nodes() + fun() -> + case LocalResult = erlang:apply(?MODULE, Func, Args) of + ResParttern -> + case rpc:multicall(nodes(), ?MODULE, Func, Args, 5000) of + {ResL, []} -> + Filter = fun + (ResParttern) -> false; + ({badrpc, {'EXIT', {undef, [{?MODULE, Func0, _, []}]}}}) + when Func0 =:= Func -> false; + (_) -> true + end, + case lists:filter(Filter, ResL) of + [] -> LocalResult; + ErrL -> {error, ErrL} + end; + {ResL, BadNodes} -> + {error, {failed_on_nodes, BadNodes, ResL}} + end; + ErrorResult -> + {error, ErrorResult} + end + end()). + +-define(SAFE_CALL(_EXP_), + ?SAFE_CALL(_EXP_, _ = do_nothing)). + +-define(SAFE_CALL(_EXP_, _EXP_ON_FAIL_), + fun() -> + try (_EXP_) + catch _EXCLASS_:_EXCPTION_:_ST_ -> + _EXP_ON_FAIL_, + {error, {_EXCLASS_, _EXCPTION_, _ST_}} + end + end()). \ No newline at end of file diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config new file mode 100644 index 000000000..83ac89bc3 --- /dev/null +++ b/apps/emqx_resource/rebar.config @@ -0,0 +1,14 @@ +{erl_opts, [ debug_info + %, {d, 'RESOURCE_DEBUG'} + ]}. + +{erl_first_files, ["src/emqx_resource_transform.erl"]}. + +{extra_src_dirs, ["examples"]}. + +{deps, [ {hocon, {git, "https://github.com/emqx/hocon", {branch, "master"}}} + , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} + , {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} + ]}. + diff --git a/apps/emqx_resource/scripts/elvis-check.sh b/apps/emqx_resource/scripts/elvis-check.sh new file mode 100755 index 000000000..3fae0f191 --- /dev/null +++ b/apps/emqx_resource/scripts/elvis-check.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +ELVIS_VERSION='1.0.0-emqx-2' + +elvis_version="${2:-$ELVIS_VERSION}" + +echo "elvis -v: $elvis_version" + +if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx-[0-9]+')" != "$elvis_version" ]; then + curl -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis" + chmod +x ./elvis +fi + +./elvis rock --config elvis.config + diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src new file mode 100644 index 000000000..af9f48cc6 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -0,0 +1,17 @@ +{application, emqx_resource, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_resource_app, []}}, + {applications, + [kernel, + stdlib, + gproc, + hocon + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl new file mode 100644 index 000000000..b9107d328 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -0,0 +1,274 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource). + +-include("emqx_resource.hrl"). +-include("emqx_resource_utils.hrl"). + +%% APIs for resource types + +-export([ get_type/1 + , list_types/0 + , list_types_verbose/0 + ]). + +-export([ discover_resource_mods/0 + , is_resource_mod/1 + , call_instance/2 + ]). + +-export([ query_success/1 + , query_failed/1 + ]). + +%% APIs for instances + +-export([ parse_config/2 + , resource_type_from_str/1 + ]). + +%% Sync resource instances and files +%% provisional solution: rpc:multical to all the nodes for creating/updating/removing +%% todo: replicate operations +-export([ create/3 %% store the config and start the instance + , create_dry_run/3 %% run start/2, health_check/2 and stop/1 sequentially + , update/4 %% update the config, stop the old instance and start the new one + %% it will create a new resource when the id does not exist + , remove/1 %% remove the config and stop the instance + ]). + +%% Calls to the callback module with current resource state +%% They also save the state after the call finished (except query/2,3). +-export([ restart/1 %% restart the instance. + , health_check/1 %% verify if the resource is working normally + , stop/1 %% stop the instance + , query/2 %% query the instance + , query/3 %% query the instance with after_query() + ]). + +%% Direct calls to the callback module +-export([ call_start/3 %% start the instance + , call_health_check/3 %% verify if the resource is working normally + , call_stop/3 %% stop the instance + , call_config_merge/4 %% merge the config when updating + ]). + +-export([ list_instances/0 %% list all the instances, id only. + , list_instances_verbose/0 %% list all the instances + , get_instance/1 %% return the data of the instance + , get_instance_by_type/1 %% return all the instances of the same resource type + , load_instances/1 %% load instances from config files + % , dependents/1 + % , inc_counter/2 %% increment the counter of the instance + % , inc_counter/3 %% increment the counter by a given integer + ]). + +-define(EXT, "*.spec"). + +-optional_callbacks([ on_query/4 + , on_health_check/2 + , on_api_reply_format/1 + , on_config_merge/3 + ]). + +-callback on_api_reply_format(resource_data()) -> map(). + +-callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). + +%% when calling emqx_resource:start/1 +-callback on_start(instance_id(), resource_config()) -> + {ok, resource_state()} | {error, Reason :: term()}. + +%% when calling emqx_resource:stop/1 +-callback on_stop(instance_id(), resource_state()) -> term(). + +%% when calling emqx_resource:query/3 +-callback on_query(instance_id(), Request :: term(), after_query(), resource_state()) -> term(). + +%% when calling emqx_resource:health_check/2 +-callback on_health_check(instance_id(), resource_state()) -> + {ok, resource_state()} | {error, Reason:: term(), resource_state()}. + +%% load specs and return the loaded resources this time. +-spec list_types_verbose() -> [resource_spec()]. +list_types_verbose() -> + [get_spec(Mod) || Mod <- list_types()]. + +-spec list_types() -> [module()]. +list_types() -> + discover_resource_mods(). + +-spec get_type(module()) -> {ok, resource_spec()} | {error, not_found}. +get_type(Mod) -> + case is_resource_mod(Mod) of + true -> {ok, get_spec(Mod)}; + false -> {error, not_found} + end. + +-spec get_spec(module()) -> resource_spec(). +get_spec(Mod) -> + maps:put(<<"resource_type">>, Mod, Mod:emqx_resource_schema()). + +-spec discover_resource_mods() -> [module()]. +discover_resource_mods() -> + [Mod || {Mod, _} <- code:all_loaded(), is_resource_mod(Mod)]. + +-spec is_resource_mod(module()) -> boolean(). +is_resource_mod(Mod) -> + erlang:function_exported(Mod, emqx_resource_schema, 0). + +-spec query_success(after_query()) -> ok. +query_success(undefined) -> ok; +query_success({{OnSucc, Args}, _}) -> + safe_apply(OnSucc, Args). + +-spec query_failed(after_query()) -> ok. +query_failed(undefined) -> ok; +query_failed({_, {OnFailed, Args}}) -> + safe_apply(OnFailed, Args). + +%% ================================================================================= +%% APIs for resource instances +%% ================================================================================= +-spec create(instance_id(), resource_type(), resource_config()) -> + {ok, resource_data()} | {error, Reason :: term()}. +create(InstId, ResourceType, Config) -> + ?CLUSTER_CALL(call_instance, [InstId, {create, InstId, ResourceType, Config}], {ok, _}). + +-spec create_dry_run(instance_id(), resource_type(), resource_config()) -> + ok | {error, Reason :: term()}. +create_dry_run(InstId, ResourceType, Config) -> + ?CLUSTER_CALL(call_instance, [InstId, {create_dry_run, InstId, ResourceType, Config}]). + +-spec update(instance_id(), resource_type(), resource_config(), term()) -> + {ok, resource_data()} | {error, Reason :: term()}. +update(InstId, ResourceType, Config, Params) -> + ?CLUSTER_CALL(call_instance, [InstId, {update, InstId, ResourceType, Config, Params}], {ok, _}). + +-spec remove(instance_id()) -> ok | {error, Reason :: term()}. +remove(InstId) -> + ?CLUSTER_CALL(call_instance, [InstId, {remove, InstId}]). + +-spec query(instance_id(), Request :: term()) -> Result :: term(). +query(InstId, Request) -> + query(InstId, Request, undefined). + +%% same to above, also defines what to do when the Module:on_query success or failed +%% it is the duty of the Moudle to apply the `after_query()` functions. +-spec query(instance_id(), Request :: term(), after_query()) -> Result :: term(). +query(InstId, Request, AfterQuery) -> + case get_instance(InstId) of + {ok, #{mod := Mod, state := ResourceState}} -> + %% the resource state is readonly to Moudle:on_query/4 + %% and the `after_query()` functions should be thread safe + Mod:on_query(InstId, Request, AfterQuery, ResourceState); + {error, Reason} -> + error({get_instance, {InstId, Reason}}) + end. + +-spec restart(instance_id()) -> ok | {error, Reason :: term()}. +restart(InstId) -> + call_instance(InstId, {restart, InstId}). + +-spec stop(instance_id()) -> ok | {error, Reason :: term()}. +stop(InstId) -> + call_instance(InstId, {stop, InstId}). + +-spec health_check(instance_id()) -> ok | {error, Reason :: term()}. +health_check(InstId) -> + call_instance(InstId, {health_check, InstId}). + +-spec get_instance(instance_id()) -> {ok, resource_data()} | {error, Reason :: term()}. +get_instance(InstId) -> + emqx_resource_instance:lookup(InstId). + +-spec list_instances() -> [instance_id()]. +list_instances() -> + [Id || #{id := Id} <- list_instances_verbose()]. + +-spec list_instances_verbose() -> [resource_data()]. +list_instances_verbose() -> + emqx_resource_instance:list_all(). + +-spec get_instance_by_type(module()) -> [resource_data()]. +get_instance_by_type(ResourceType) -> + emqx_resource_instance:lookup_by_type(ResourceType). + +-spec load_instances(Dir :: string()) -> ok. +load_instances(Dir) -> + emqx_resource_instance:load(Dir). + +-spec call_start(instance_id(), module(), resource_config()) -> + {ok, resource_state()} | {error, Reason :: term()}. +call_start(InstId, Mod, Config) -> + ?SAFE_CALL(Mod:on_start(InstId, Config)). + +-spec call_health_check(instance_id(), module(), resource_state()) -> + {ok, resource_state()} | {error, Reason:: term(), resource_state()}. +call_health_check(InstId, Mod, ResourceState) -> + ?SAFE_CALL(Mod:on_health_check(InstId, ResourceState)). + +-spec call_stop(instance_id(), module(), resource_state()) -> term(). +call_stop(InstId, Mod, ResourceState) -> + ?SAFE_CALL(Mod:on_stop(InstId, ResourceState)). + +-spec call_config_merge(module(), resource_config(), resource_config(), term()) -> + resource_config(). +call_config_merge(Mod, OldConfig, NewConfig, Params) -> + ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)). + +-spec parse_config(resource_type(), binary() | term()) -> + {ok, resource_config()} | {error, term()}. +parse_config(ResourceType, RawConfig) when is_binary(RawConfig) -> + case hocon:binary(RawConfig, #{format => richmap}) of + {ok, MapConfig} -> + do_parse_config(ResourceType, MapConfig); + Error -> Error + end; +parse_config(ResourceType, RawConfigTerm) -> + parse_config(ResourceType, jsx:encode(#{<<"config">> => RawConfigTerm})). + +-spec do_parse_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. +do_parse_config(ResourceType, MapConfig) -> + case ?SAFE_CALL(hocon_schema:generate(ResourceType, MapConfig)) of + {error, Reason} -> {error, Reason}; + Config -> + InstConf = maps:from_list(proplists:get_value(config, Config)), + {ok, InstConf} + end. + +%% ================================================================================= + +-spec resource_type_from_str(string()) -> {ok, resource_type()} | {error, term()}. +resource_type_from_str(ResourceType) -> + try Mod = list_to_existing_atom(str(ResourceType)), + case emqx_resource:is_resource_mod(Mod) of + true -> {ok, Mod}; + false -> {error, {invalid_resource, Mod}} + end + catch error:badarg -> + {error, {not_found, ResourceType}} + end. + +call_instance(InstId, Query) -> + emqx_resource_instance:hash_call(InstId, Query). + +safe_apply(Func, Args) -> + ?SAFE_CALL(erlang:apply(Func, Args)). + +str(S) when is_binary(S) -> binary_to_list(S); +str(S) when is_list(S) -> S. diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl new file mode 100644 index 000000000..82652039d --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -0,0 +1,64 @@ +-module(emqx_resource_api). + +-export([ get_all/3 + , get/3 + , put/3 + , delete/3 + ]). +get_all(Mod, _Binding, _Params) -> + {200, #{code => 0, data => + [format_data(Mod, Data) || Data <- emqx_resource:list_instances_verbose()]}}. + +get(Mod, #{id := Id}, _Params) -> + case emqx_resource:get_instance(stringnify(Id)) of + {ok, Data} -> + {200, #{code => 0, data => format_data(Mod, Data)}}; + {error, not_found} -> + {404, #{code => 102, message => {resource_instance_not_found, stringnify(Id)}}} + end. + +put(Mod, #{id := Id}, Params) -> + ConfigParams = proplists:get_value(<<"config">>, Params), + ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params), + case emqx_resource:resource_type_from_str(ResourceTypeStr) of + {ok, ResourceType} -> + do_put(Mod, stringnify(Id), ConfigParams, ResourceType, Params); + {error, Reason} -> + {404, #{code => 102, message => stringnify(Reason)}} + end. + +do_put(Mod, Id, ConfigParams, ResourceType, Params) -> + case emqx_resource:parse_config(ResourceType, ConfigParams) of + {ok, Config} -> + case emqx_resource:update(Id, ResourceType, Config, Params) of + {ok, Data} -> + {200, #{code => 0, data => format_data(Mod, Data)}}; + {error, Reason} -> + {500, #{code => 102, message => stringnify(Reason)}} + end; + {error, Reason} -> + {400, #{code => 108, message => stringnify(Reason)}} + end. + +delete(_Mod, #{id := Id}, _Params) -> + case emqx_resource:remove(stringnify(Id)) of + ok -> {200, #{code => 0, data => #{}}}; + {error, Reason} -> + {500, #{code => 102, message => stringnify(Reason)}} + end. + +format_data(Mod, Data) -> + case erlang:function_exported(Mod, on_api_reply_format, 1) of + false -> + default_api_reply_format(Data); + true -> + Mod:on_api_reply_format(Data) + end. + +default_api_reply_format(#{id := Id, status := Status, config := Config}) -> + #{node => node(), id => Id, status => Status, config => Config}. + +stringnify(Bin) when is_binary(Bin) -> Bin; +stringnify(Str) when is_list(Str) -> list_to_binary(Str); +stringnify(Reason) -> + iolist_to_binary(io_lib:format("~p", [Reason])). diff --git a/apps/emqx_resource/src/emqx_resource_app.erl b/apps/emqx_resource/src/emqx_resource_app.erl new file mode 100644 index 000000000..e3ef9f3d2 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_app.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_app). + +-behaviour(application). + +-include("emqx_resource.hrl"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_resource_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl new file mode 100644 index 000000000..d47ec18df --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -0,0 +1,294 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_instance). + +-behaviour(gen_server). + +-include("emqx_resource.hrl"). +-include("emqx_resource_utils.hrl"). + +-export([start_link/2]). + +%% load resource instances from *.conf files +-export([ load/1 + , lookup/1 + , list_all/0 + , lookup_by_type/1 + ]). + +-export([ hash_call/2 + , hash_call/3 + ]). + +%% gen_server Callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, {worker_pool, worker_id}). + +-type state() :: #state{}. + +%%------------------------------------------------------------------------------ +%% Start the registry +%%------------------------------------------------------------------------------ + +start_link(Pool, Id) -> + gen_server:start_link({local, proc_name(?MODULE, Id)}, + ?MODULE, {Pool, Id}, []). + +%% call the worker by the hash of resource-instance-id, to make sure we always handle +%% operations on the same instance in the same worker. +hash_call(InstId, Request) -> + hash_call(InstId, Request, infinity). + +hash_call(InstId, Request, Timeout) -> + gen_server:call(pick(InstId), Request, Timeout). + +-spec lookup(instance_id()) -> {ok, resource_data()} | {error, Reason :: term()}. +lookup(InstId) -> + case ets:lookup(emqx_resource_instance, InstId) of + [] -> {error, not_found}; + [{_, Data}] -> {ok, Data#{id => InstId}} + end. + +force_lookup(InstId) -> + {ok, Data} = lookup(InstId), + Data. + +-spec list_all() -> [resource_data()]. +list_all() -> + [Data#{id => Id} || {Id, Data} <- ets:tab2list(emqx_resource_instance)]. + +-spec lookup_by_type(module()) -> [resource_data()]. +lookup_by_type(ResourceType) -> + [Data || #{mod := Mod} = Data <- list_all() + , Mod =:= ResourceType]. + +-spec load(Dir :: string()) -> ok. +load(Dir) -> + lists:foreach(fun load_file/1, filelib:wildcard(filename:join([Dir, "*.conf"]))). + +load_file(File) -> + case ?SAFE_CALL(hocon_token:read(File)) of + {error, Reason} -> + logger:error("load resource from ~p failed: ~p", [File, Reason]); + RawConfig -> + case hocon:binary(RawConfig, #{format => map}) of + {ok, #{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr, + <<"config">> := MapConfig}} -> + case emqx_resource:resource_type_from_str(ResourceTypeStr) of + {ok, ResourceType} -> + parse_and_load_config(Id, ResourceType, MapConfig); + {error, Reason} -> + logger:error("no such resource type: ~s, ~p", + [ResourceTypeStr, Reason]) + end; + {error, Reason} -> + logger:error("load resource from ~p failed: ~p", [File, Reason]) + end + end. + +parse_and_load_config(InstId, ResourceType, MapConfig) -> + case emqx_resource:parse_config(ResourceType, MapConfig) of + {error, Reason} -> + logger:error("parse config for resource ~p of type ~p failed: ~p", + [InstId, ResourceType, Reason]); + {ok, InstConf} -> + create_instance_local(InstId, ResourceType, InstConf) + end. + +create_instance_local(InstId, ResourceType, InstConf) -> + case do_create(InstId, ResourceType, InstConf) of + {ok, Data} -> + logger:debug("created ~p resource instance: ~p from config: ~p, Data: ~p", + [ResourceType, InstId, InstConf, Data]); + {error, Reason} -> + logger:error("create ~p resource instance: ~p failed: ~p, config: ~p", + [ResourceType, InstId, Reason, InstConf]) + end. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +-spec init({atom(), integer()}) -> + {ok, State :: state()} | {ok, State :: state(), timeout() | hibernate | {continue, term()}} | + {stop, Reason :: term()} | ignore. +init({Pool, Id}) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #state{worker_pool = Pool, worker_id = Id}}. + +handle_call({create, InstId, ResourceType, Config}, _From, State) -> + {reply, do_create(InstId, ResourceType, Config), State}; + +handle_call({create_dry_run, InstId, ResourceType, Config}, _From, State) -> + {reply, do_create_dry_run(InstId, ResourceType, Config), State}; + +handle_call({update, InstId, ResourceType, Config, Params}, _From, State) -> + {reply, do_update(InstId, ResourceType, Config, Params), State}; + +handle_call({remove, InstId}, _From, State) -> + {reply, do_remove(InstId), State}; + +handle_call({restart, InstId}, _From, State) -> + {reply, do_restart(InstId), State}; + +handle_call({stop, InstId}, _From, State) -> + {reply, do_stop(InstId), State}; + +handle_call({health_check, InstId}, _From, State) -> + {reply, do_health_check(InstId), State}; + +handle_call(Req, _From, State) -> + logger:error("Received unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #state{worker_pool = Pool, worker_id = Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ + +do_update(InstId, ResourceType, NewConfig, Params) -> + case lookup(InstId) of + {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> + Config = emqx_resource:call_config_merge(ResourceType, OldConfig, + NewConfig, Params), + case do_create_dry_run(InstId, ResourceType, Config) of + ok -> + do_remove(ResourceType, InstId, ResourceState), + do_create(InstId, ResourceType, Config); + Error -> + Error + end; + {ok, #{mod := Mod}} when Mod =/= ResourceType -> + {error, updating_to_incorrect_resource_type}; + {error, not_found} -> + do_create(InstId, ResourceType, NewConfig) + end. + +do_create(InstId, ResourceType, Config) -> + case lookup(InstId) of + {ok, _} -> {error, already_created}; + _ -> + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState} -> + ets:insert(emqx_resource_instance, {InstId, + #{mod => ResourceType, config => Config, + state => ResourceState, status => stopped}}), + _ = do_health_check(InstId), + {ok, force_lookup(InstId)}; + {error, Reason} -> + logger:error("start ~s resource ~s failed: ~p", [ResourceType, InstId, Reason]), + {error, Reason} + end + end. + +do_create_dry_run(InstId, ResourceType, Config) -> + case emqx_resource:call_start(InstId, ResourceType, Config) of + {ok, ResourceState0} -> + Return = case emqx_resource:call_health_check(InstId, ResourceType, ResourceState0) of + {ok, ResourceState1} -> ok; + {error, Reason, ResourceState1} -> + {error, Reason} + end, + _ = emqx_resource:call_stop(InstId, ResourceType, ResourceState1), + Return; + {error, Reason} -> + {error, Reason} + end. + +do_remove(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState}} -> + do_remove(Mod, InstId, ResourceState); + Error -> + Error + end. + +do_remove(Mod, InstId, ResourceState) -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ets:delete(emqx_resource_instance, InstId), + ok. + +do_restart(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState, config := Config} = Data} -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + case emqx_resource:call_start(InstId, Mod, Config) of + {ok, ResourceState} -> + ets:insert(emqx_resource_instance, + {InstId, Data#{state => ResourceState, status => started}}), + ok; + {error, Reason} -> + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), + {error, Reason} + end; + Error -> + Error + end. + +do_stop(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState} = Data} -> + _ = emqx_resource:call_stop(InstId, Mod, ResourceState), + ets:insert(emqx_resource_instance, {InstId, Data#{status => stopped}}), + ok; + Error -> + Error + end. + +do_health_check(InstId) -> + case lookup(InstId) of + {ok, #{mod := Mod, state := ResourceState0} = Data} -> + case emqx_resource:call_health_check(InstId, Mod, ResourceState0) of + {ok, ResourceState1} -> + ets:insert(emqx_resource_instance, + {InstId, Data#{status => started, state => ResourceState1}}), + ok; + {error, Reason, ResourceState1} -> + logger:error("health check for ~p failed: ~p", [InstId, Reason]), + ets:insert(emqx_resource_instance, + {InstId, Data#{status => stopped, state => ResourceState1}}), + {error, Reason} + end; + Error -> + Error + end. + +%%------------------------------------------------------------------------------ +%% internal functions +%%------------------------------------------------------------------------------ + +proc_name(Mod, Id) -> + list_to_atom(lists:concat([Mod, "_", Id])). + +pick(InstId) -> + gproc_pool:pick_worker(emqx_resource_instance, InstId). diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl new file mode 100644 index 000000000..275de6dec --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -0,0 +1,48 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_resource top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_resource_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(RESOURCE_INST_MOD, emqx_resource_instance). +-define(POOL_SIZE, 64). %% set a very large pool size in case all the workers busy + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + TabOpts = [named_table, set, public, {read_concurrency, true}], + _ = ets:new(emqx_resource_instance, TabOpts), + + SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, + Pool = ?RESOURCE_INST_MOD, + Mod = ?RESOURCE_INST_MOD, + ensure_pool(Pool, hash, [{size, ?POOL_SIZE}]), + {ok, {SupFlags, [ + begin + ensure_pool_worker(Pool, {Pool, Idx}, Idx), + #{id => {Mod, Idx}, + start => {Mod, start_link, [Pool, Idx]}, + restart => transient, + shutdown => 5000, type => worker, modules => [Mod]} + end || Idx <- lists:seq(1, ?POOL_SIZE)]}}. + +%% internal functions +ensure_pool(Pool, Type, Opts) -> + try gproc_pool:new(Pool, Type, Opts) + catch + error:exists -> ok + end. + +ensure_pool_worker(Pool, Name, Slot) -> + try gproc_pool:add_worker(Pool, Name, Slot) + catch + error:exists -> ok + end. \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl new file mode 100644 index 000000000..844350278 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -0,0 +1,99 @@ +-module(emqx_resource_transform). + +-include_lib("syntax_tools/include/merl.hrl"). + +-export([parse_transform/2]). + +parse_transform(Forms, _Opts) -> + Mod = hd([M || {attribute, _, module, M} <- Forms]), + AST = trans(Mod, proplists:delete(eof, Forms)), + debug_print(Mod, AST), + AST. + +-ifdef(RESOURCE_DEBUG). + +debug_print(Mod, Ts) -> + {ok, Io} = file:open("./" ++ atom_to_list(Mod) ++ ".trans.erl", [write]), + do_debug_print(Io, Ts), + file:close(Io). + +do_debug_print(Io, Ts) when is_list(Ts) -> + lists:foreach(fun(T) -> do_debug_print(Io, T) end, Ts); +do_debug_print(Io, T) -> + io:put_chars(Io, erl_prettypr:format(merl:tree(T))), + io:nl(Io). +-else. +debug_print(_Mod, _AST) -> + ok. +-endif. + +trans(Mod, Forms) -> + forms(Mod, Forms) ++ [erl_syntax:revert(erl_syntax:eof_marker())]. + +forms(Mod, [F0 | Fs0]) -> + case form(Mod, F0) of + {CurrForm, AppendedForms} -> + CurrForm ++ forms(Mod, Fs0) ++ AppendedForms; + {AHeadForms, CurrForm, AppendedForms} -> + AHeadForms ++ CurrForm ++ forms(Mod, Fs0) ++ AppendedForms + end; +forms(_, []) -> []. + +form(Mod, Form) -> + case Form of + ?Q("-emqx_resource_api_path('@Path').") -> + {fix_spec_attrs() ++ fix_api_attrs(erl_syntax:concrete(Path)) ++ fix_api_exports(), + [], + fix_spec_funcs(Mod) ++ fix_api_funcs(Mod)}; + _ -> + %io:format("---other form: ~p~n", [Form]), + {[], [Form], []} + end. + +fix_spec_attrs() -> + [ ?Q("-export([emqx_resource_schema/0]).") + , ?Q("-export([structs/0]).") + , ?Q("-behaviour(hocon_schema).") + ]. +fix_spec_funcs(_Mod) -> + [ (?Q("emqx_resource_schema() -> <<\"demo_swagger_schema\">>.")) + , ?Q("structs() -> [\"config\"].") + ]. + +fix_api_attrs(Path0) -> + BaseName = filename:basename(Path0), + Path = "/" ++ BaseName, + [erl_syntax:revert( + erl_syntax:attribute(?Q("rest_api"), [ + erl_syntax:abstract(#{ + name => list_to_atom(Name ++ "_log_tracers"), + method => Method, + path => mk_path(Path, WithId), + func => Func, + descr => Name ++ " the " ++ BaseName})])) + || {Name, Method, WithId, Func} <- [ + {"list", 'GET', noid, api_get_all}, + {"get", 'GET', id, api_get}, + {"update", 'PUT', id, api_put}, + {"delete", 'DELETE', id, api_delete}]]. + +fix_api_exports() -> + [?Q("-export([api_get_all/2, api_get/2, api_put/2, api_delete/2]).")]. + +fix_api_funcs(Mod) -> + [erl_syntax:revert(?Q( + "api_get_all(Binding, Params) -> + emqx_resource_api:get_all('@Mod@', Binding, Params).")), + erl_syntax:revert(?Q( + "api_get(Binding, Params) -> + emqx_resource_api:get('@Mod@', Binding, Params).")), + erl_syntax:revert(?Q( + "api_put(Binding, Params) -> + emqx_resource_api:put('@Mod@', Binding, Params).")), + erl_syntax:revert(?Q( + "api_delete(Binding, Params) -> + emqx_resource_api:delete('@Mod@', Binding, Params).")) + ]. + +mk_path(Path, id) -> Path ++ "/:bin:id"; +mk_path(Path, noid) -> Path. diff --git a/apps/emqx_resource/src/emqx_resource_uitils.erl b/apps/emqx_resource/src/emqx_resource_uitils.erl new file mode 100644 index 000000000..b49eeb927 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_uitils.erl @@ -0,0 +1 @@ +-module(emqx_resource_uitils). \ No newline at end of file diff --git a/apps/emqx_resource/src/emqx_resource_validator.erl b/apps/emqx_resource/src/emqx_resource_validator.erl new file mode 100644 index 000000000..e9517f160 --- /dev/null +++ b/apps/emqx_resource/src/emqx_resource_validator.erl @@ -0,0 +1,63 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_resource_validator). + +-export([ min/2 + , max/2 + , equals/2 + , enum/1 + , required/1 + ]). + +max(Type, Max) -> + limit(Type, '=<', Max). + +min(Type, Min) -> + limit(Type, '>=', Min). + +equals(Type, Expected) -> + limit(Type, '==', Expected). + +enum(Items) -> + fun(Value) -> + return(lists:member(Value, Items), + err_limit({enum, {is_member_of, Items}, {got, Value}})) + end. + +required(ErrMsg) -> + fun(undefined) -> {error, ErrMsg}; + (_) -> ok + end. + +limit(Type, Op, Expected) -> + L = len(Type), + fun(Value) -> + Got = L(Value), + return(erlang:Op(Got, Expected), + err_limit({Type, {Op, Expected}, {got, Got}})) + end. + +len(array) -> fun erlang:length/1; +len(string) -> fun string:length/1; +len(_Type) -> fun(Val) -> Val end. + +err_limit({Type, {Op, Expected}, {got, Got}}) -> + io_lib:format("Expect the ~s value ~s ~p but got: ~p", [Type, Op, Expected, Got]). + +return(true, _) -> ok; +return(false, Error) -> + {error, Error}. From bc83bed7e865c245aa03955235774de483f3ec05 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 28 May 2021 22:02:33 +0800 Subject: [PATCH 41/58] feat(emqx_resource): add licence headers to source code files --- apps/emqx_resource/LICENSE | 191 ------------------ apps/emqx_resource/include/emqx_resource.hrl | 15 ++ .../include/emqx_resource_behaviour.hrl | 15 ++ .../include/emqx_resource_utils.hrl | 15 ++ apps/emqx_resource/rebar.config | 4 +- apps/emqx_resource/src/emqx_resource_api.erl | 15 ++ .../src/emqx_resource_instance.erl | 1 - apps/emqx_resource/src/emqx_resource_sup.erl | 20 +- .../src/emqx_resource_transform.erl | 15 ++ .../src/emqx_resource_uitils.erl | 15 ++ rebar.config.erl | 1 + 11 files changed, 107 insertions(+), 200 deletions(-) delete mode 100644 apps/emqx_resource/LICENSE diff --git a/apps/emqx_resource/LICENSE b/apps/emqx_resource/LICENSE deleted file mode 100644 index 2f97fdd03..000000000 --- a/apps/emqx_resource/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2021, Shawn <506895667@qq.com>. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 429b83c45..1f75b453e 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -type resource_type() :: module(). -type instance_id() :: binary(). -type resource_config() :: jsx:json_term(). diff --git a/apps/emqx_resource/include/emqx_resource_behaviour.hrl b/apps/emqx_resource/include/emqx_resource_behaviour.hrl index bf0f1cd2e..bb4f18b55 100644 --- a/apps/emqx_resource/include/emqx_resource_behaviour.hrl +++ b/apps/emqx_resource/include/emqx_resource_behaviour.hrl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). -compile({parse_transform, emqx_resource_transform}). diff --git a/apps/emqx_resource/include/emqx_resource_utils.hrl b/apps/emqx_resource/include/emqx_resource_utils.hrl index dd1517428..a20a17e89 100644 --- a/apps/emqx_resource/include/emqx_resource_utils.hrl +++ b/apps/emqx_resource/include/emqx_resource_utils.hrl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -define(CLUSTER_CALL(Func, Args), ?CLUSTER_CALL(Func, Args, ok)). -define(CLUSTER_CALL(Func, Args, ResParttern), diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 83ac89bc3..2505d2e5c 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -4,11 +4,9 @@ {erl_first_files, ["src/emqx_resource_transform.erl"]}. -{extra_src_dirs, ["examples"]}. +%{extra_src_dirs, ["examples"]}. {deps, [ {hocon, {git, "https://github.com/emqx/hocon", {branch, "master"}}} - , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}} - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} ]}. diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index 82652039d..cc32d8a11 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_resource_api). -export([ get_all/3 diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index d47ec18df..a78a2e15a 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -13,7 +13,6 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- - -module(emqx_resource_instance). -behaviour(gen_server). diff --git a/apps/emqx_resource/src/emqx_resource_sup.erl b/apps/emqx_resource/src/emqx_resource_sup.erl index 275de6dec..22984b940 100644 --- a/apps/emqx_resource/src/emqx_resource_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_sup.erl @@ -1,8 +1,18 @@ -%%%------------------------------------------------------------------- -%% @doc emqx_resource top level supervisor. -%% @end -%%%------------------------------------------------------------------- - +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_resource_sup). -behaviour(supervisor). diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index 844350278..23e14013c 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -1,3 +1,18 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_resource_transform). -include_lib("syntax_tools/include/merl.hrl"). diff --git a/apps/emqx_resource/src/emqx_resource_uitils.erl b/apps/emqx_resource/src/emqx_resource_uitils.erl index b49eeb927..ab3f6dd1e 100644 --- a/apps/emqx_resource/src/emqx_resource_uitils.erl +++ b/apps/emqx_resource/src/emqx_resource_uitils.erl @@ -1 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- -module(emqx_resource_uitils). \ No newline at end of file diff --git a/rebar.config.erl b/rebar.config.erl index c4c0419b9..e3a6c427b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -273,6 +273,7 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon + , emqx_resource , emqx_rule_engine , emqx_sasl ] From 9d1f6ccc4b1ef4c4ff8189d25a468a2c0a34f308 Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Fri, 28 May 2021 14:54:22 +0800 Subject: [PATCH 42/58] chore: ekka support etcd v3 --- etc/emqx.conf | 7 +++++++ priv/emqx.schema | 5 +++++ rebar.config | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/etc/emqx.conf b/etc/emqx.conf index 7b27f100c..65d8ec58b 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -109,6 +109,13 @@ cluster.autoclean = 5m ## Value: String ## cluster.etcd.server = "http://127.0.0.1:2379" +## Etcd api version +## +## Value: Enum +## - v2 +## - v3 +## cluster.etcd.version = v3 + ## The prefix helps build nodes path in etcd. Each node in the cluster ## will create a path in etcd: v2/keys/// ## diff --git a/priv/emqx.schema b/priv/emqx.schema index f107798a2..37679cf59 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -103,6 +103,10 @@ {datatype, string} ]}. +{mapping, "cluster.etcd.version", "ekka.cluster_discovery", [ + {datatype, {enum, [v2, v3]}} +]}. + {mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ {datatype, string} ]}. @@ -180,6 +184,7 @@ end, Options) end, [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, + {version, cuttlefish:conf_get("cluster.etcd.version", Conf, v3)}, {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}, {ssl_options, SslOpts(Conf)}]; diff --git a/rebar.config b/rebar.config index 2059854dd..2ee94ff04 100644 --- a/rebar.config +++ b/rebar.config @@ -41,7 +41,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.9.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} From d7755df48b824b4555501c77df44c822dcb5aad3 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 31 May 2021 10:35:29 +0800 Subject: [PATCH 43/58] feat(emqx_resource): make with emqx --- apps/emqx_resource/etc/emqx_resource.conf | 3 +++ apps/emqx_resource/priv/emqx_resource.schema | 2 ++ apps/emqx_resource/rebar.config | 6 +++++- apps/emqx_resource/src/emqx_resource_app.erl | 2 ++ data/loaded_plugins.tmpl | 1 + rebar.config.erl | 1 + 6 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_resource/etc/emqx_resource.conf create mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_resource/etc/emqx_resource.conf b/apps/emqx_resource/etc/emqx_resource.conf new file mode 100644 index 000000000..1f038ef8b --- /dev/null +++ b/apps/emqx_resource/etc/emqx_resource.conf @@ -0,0 +1,3 @@ +##-------------------------------------------------------------------- +## EMQ X Resource Plugin +##-------------------------------------------------------------------- diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema new file mode 100644 index 000000000..8246dc6a7 --- /dev/null +++ b/apps/emqx_resource/priv/emqx_resource.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx-resource config mapping diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 2505d2e5c..089985cec 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -1,10 +1,14 @@ {erl_opts, [ debug_info + , nowarn_unused_import %, {d, 'RESOURCE_DEBUG'} ]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}. -%{extra_src_dirs, ["examples"]}. +{extra_src_dirs, ["examples"]}. + +{dialyzer, [{warnings, [unmatched_returns, error_handling]} + ]}. {deps, [ {hocon, {git, "https://github.com/emqx/hocon", {branch, "master"}}} , {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} diff --git a/apps/emqx_resource/src/emqx_resource_app.erl b/apps/emqx_resource/src/emqx_resource_app.erl index e3ef9f3d2..d2c499490 100644 --- a/apps/emqx_resource/src/emqx_resource_app.erl +++ b/apps/emqx_resource/src/emqx_resource_app.erl @@ -20,6 +20,8 @@ -include("emqx_resource.hrl"). +-emqx_plugin(?MODULE). + -export([start/2, stop/1]). start(_StartType, _StartArgs) -> diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..236eeaa95 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,4 +5,5 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. +{emqx_resource, {{enable_plugin_emqx_resource}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index e3a6c427b..f46dfa0d8 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -180,6 +180,7 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} + , {enable_plugin_emqx_resource, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} From 696d4a4e584c4b4bcbe59b0667d60936a09b7b23 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Mon, 31 May 2021 12:14:25 +0800 Subject: [PATCH 44/58] fix(emqx_resource): some dialyzer complaints --- apps/emqx_resource/rebar.config | 2 ++ apps/emqx_resource/src/emqx_resource_instance.erl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 089985cec..148af92bd 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -7,6 +7,8 @@ {extra_src_dirs, ["examples"]}. +%% try to override the dialyzer 'race_conditions' defined in the top-level dir, +%% but it doesn't work {dialyzer, [{warnings, [unmatched_returns, error_handling]} ]}. diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index a78a2e15a..ca5e4829e 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -175,6 +175,8 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ +%% suppress the race condition check, as these functions are protected in gproc workers +-dialyzer({nowarn_function, [do_update/4, do_create/3, do_restart/1, do_stop/1, do_health_check/1]}). do_update(InstId, ResourceType, NewConfig, Params) -> case lookup(InstId) of {ok, #{mod := ResourceType, state := ResourceState, config := OldConfig}} -> From d61e931d886b96b317b8a96bbdc5706bb9c44113 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 25 May 2021 15:14:43 +0900 Subject: [PATCH 45/58] fix(emqx_logger_jsonfmt): attribute import before function definitions --- src/emqx_logger_jsonfmt.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/emqx_logger_jsonfmt.erl b/src/emqx_logger_jsonfmt.erl index 31e4c2fda..fdc1a8bb3 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/src/emqx_logger_jsonfmt.erl @@ -32,6 +32,9 @@ -export([format/2]). -ifdef(TEST). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -export([report_cb_1/1, report_cb_2/2, report_cb_crash/2]). -endif. @@ -220,8 +223,6 @@ json_key(Term) -> end. -ifdef(TEST). --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). no_crash_test_() -> Opts = [{numtests, 1000}, {to_file, user}], From 4b174b027700f4d8185c6482e49df176aee06188 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 25 May 2021 17:25:25 +0900 Subject: [PATCH 46/58] feat(conf): use hocon schema --- .ci/docker-compose-file/conf.env | 1 + bin/emqx | 31 +- etc/emqx.conf | 22 +- rebar.config | 3 +- rebar.config.erl | 5 +- src/emqx_schema.erl | 1443 ++++++++++++++++++++++++++++++ test/emqx_listeners_SUITE.erl | 5 +- 7 files changed, 1477 insertions(+), 33 deletions(-) create mode 100644 src/emqx_schema.erl diff --git a/.ci/docker-compose-file/conf.env b/.ci/docker-compose-file/conf.env index 93dfecd2b..0b1b7c512 100644 --- a/.ci/docker-compose-file/conf.env +++ b/.ci/docker-compose-file/conf.env @@ -11,3 +11,4 @@ EMQX_AUTH__PGSQL__DATABASE=mqtt EMQX_AUTH__REDIS__SERVER=redis_server:6379 EMQX_AUTH__REDIS__PASSWORD=public CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_ +HOCON_ENV_OVERRIDE_PREFIX=EMQX_ diff --git a/bin/emqx b/bin/emqx index fd6986614..1a85f6fbb 100755 --- a/bin/emqx +++ b/bin/emqx @@ -20,8 +20,8 @@ mkdir -p "$RUNNER_LOG_DIR" # Make sure data directory exists mkdir -p "$RUNNER_DATA_DIR" -# cuttlefish try to read environment variables starting with "EMQX_" -export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_' +# hocon try to read environment variables starting with "EMQX_" +export HOCON_ENV_OVERRIDE_PREFIX='EMQX_' relx_usage() { command="$1" @@ -123,8 +123,8 @@ fi # Echo to stderr on errors echoerr() { echo "$@" 1>&2; } -# By default, use cuttlefish to generate app.config and vm.args -CUTTLEFISH="${USE_CUTTLEFISH:-yes}" +# By default, use hocon to generate app.config and vm.args +HOCON="${USE_HOCON:-yes}" SED_REPLACE="sed -i " case $(sed --help 2>&1) in @@ -202,7 +202,7 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - if [ "$CUTTLEFISH" != "yes" ]; then + if [ "$HOCON" != "yes" ]; then # Note: we have added a parameter '-vm_args' to this. It # appears redundant but it is not! the erlang vm allows us to # access all arguments to the erl command EXCEPT '-args_file', @@ -217,25 +217,26 @@ generate_config() { set +e # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + echo $HOCON_OUTPUT # shellcheck disable=SC2181 RESULT=$? set -e if [ $RESULT -gt 0 ]; then - echo "$CUTTLEFISH_OUTPUT" + echo "$HOCON_OUTPUT" exit $RESULT fi # print override from environment variables (EMQX_*) - echo "$CUTTLEFISH_OUTPUT" | sed -e '$d' - CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1) + echo "$HOCON_OUTPUT" | sed -e '$d' + CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1) - ## Merge cuttlefish generated *.args into the vm.args - CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') + ## Merge hocon generated *.args into the vm.args + HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" echo "" >> "$TMP_ARG_FILE" echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" - sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do + sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') @@ -247,7 +248,7 @@ generate_config() { fi fi done - mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE" + mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE" fi # shellcheck disable=SC2086 @@ -303,7 +304,7 @@ if [ -z "$NAME_ARG" ]; then NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')" else # for boot commands, inspect emqx.conf for node name - NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name) + NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \") fi fi if [ -z "$NODENAME" ]; then @@ -329,7 +330,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" COOKIE="${EMQX_NODE_COOKIE:-}" if [ -z "$COOKIE" ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then - COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie) + COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') else # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" diff --git a/etc/emqx.conf b/etc/emqx.conf index 65d8ec58b..c7c044c99 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -598,42 +598,42 @@ log.rotation.count = 5 ## Notice: Disable the option in production deployment! ## ## Value: true | false -allow_anonymous = true +acl.allow_anonymous = true ## Allow or deny if no ACL rules matched. ## ## Value: allow | deny -acl_nomatch = allow +acl.acl_nomatch = allow ## Default ACL File. ## ## Value: File Name -acl_file = "{{ platform_etc_dir }}/acl.conf" +acl.acl_file = "{{ platform_etc_dir }}/acl.conf" ## Whether to enable ACL cache. ## ## If enabled, ACLs roles for each client will be cached in the memory ## ## Value: on | off -enable_acl_cache = on +acl.enable_acl_cache = on ## The maximum count of ACL entries can be cached for a client. ## ## Value: Integer greater than 0 ## Default: 32 -acl_cache_max_size = 32 +acl.acl_cache_max_size = 32 ## The time after which an ACL cache entry will be deleted ## ## Value: Duration ## Default: 1 minute -acl_cache_ttl = 1m +acl.acl_cache_ttl = 1m ## The action when acl check reject current operation ## ## Value: ignore | disconnect ## Default: ignore -acl_deny_action = ignore +acl.acl_deny_action = ignore ## Specify the global flapping detect policy. ## The value is a string composed of flapping threshold, duration and banned interval. @@ -642,7 +642,7 @@ acl_deny_action = ignore ## 3. banned interval: the banned interval if a flapping is detected. ## ## Value: Integer,Duration,Duration -flapping_detect_policy = "30, 1m, 5m" +acl.flapping_detect_policy = "30, 1m, 5m" ##-------------------------------------------------------------------- ## MQTT Protocol @@ -2155,7 +2155,7 @@ listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1 ## The file to store loaded module names. ## ## Value: File -modules.loaded_file = "{{ platform_data_dir }}/loaded_modules" +module.loaded_file = "{{ platform_data_dir }}/loaded_modules" ##-------------------------------------------------------------------- ## Presence Module @@ -2204,8 +2204,8 @@ module.presence.qos = 1 ## Rewrite Module ## {rewrite, Topic, Re, Dest} -## module.rewrite.pub.rule.1 = "x/# ^x/y/(.+)$ z/y/$1" -## module.rewrite.sub.rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" +## module.rewrite.pub_rule.1 = "x/# ^x/y/(.+)$ z/y/$1" +## module.rewrite.sub_rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2" ## CONFIG_SECTION_END=modules ================================================== diff --git a/rebar.config b/rebar.config index ec78316bc..f52f5cfba 100644 --- a/rebar.config +++ b/rebar.config @@ -7,7 +7,7 @@ %% rebar.config.rendered if environment DEBUG is set. {edoc_opts, [{preprocess,true}]}. -{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, +{erl_opts, [warn_unused_vars,warn_shadow_vars, warn_obsolete_guard,compressed, {d, snk_kind, msg}]}. @@ -56,6 +56,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.2.0"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 8ac16ebbb..a72f57933 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -336,9 +336,8 @@ relx_overlay(ReleaseType) -> , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"} , {copy, "bin/nodetool", "bin/nodetool"} , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"} - , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"} - , {copy, "priv/emqx.schema", "releases/{{release_version}}/"} + , {copy, "_build/default/lib/hocon/hocon", "bin/hocon"} + , {copy, "_build/default/lib/hocon/hocon", "bin/hocon-{{release_version}}"} ] ++ case is_enterprise() of true -> ee_etc_overlay(ReleaseType); false -> etc_overlay(ReleaseType) diff --git a/src/emqx_schema.erl b/src/emqx_schema.erl new file mode 100644 index 000000000..c2a161d83 --- /dev/null +++ b/src/emqx_schema.erl @@ -0,0 +1,1443 @@ +-module(emqx_schema). + +-include_lib("typerefl/include/types.hrl"). + +-type rpc_mode() :: sync | async. +-type proto_dist() :: inet_tcp | inet6_tcp | inet_tls. +-type k8s_address_type() :: ip | dns | hostname. +-type port_discovery() :: manual | stateless. +-type acl_nomatch() :: allow | deny. +-type acl_deny_action() :: ignore | disconnect. +-type mqueue_default_priority() :: highest | lowest. +-type endpoint() :: integer() | string(). +-type listener_peer_cert_tcp() :: cn | tmp. % @todo fix +-type listener_peer_cert_ssl() :: cn | dn | crt | pem | md5. +-type mqtt_piggyback() :: single | multiple. +-type deflate_opts_level() :: none | default | best_compression | best_speed. +-type deflate_opts_strategy() :: default | filtered | huffman_only | rle. +-type context_takeover() :: takeover | no_takeover. +-type verify() :: verify_peer | verify_none. +-type session_locking_strategy() :: local | one | quorum | all. +-type shared_subscription_strategy() :: random | round_robin | sticky | hash. +-type route_lock_type() :: key | tab | global. +-type log_format_depth() :: unlimited | integer(). +-type log_size() :: atom() | bytesize(). +-type overload_kill_restart_after() :: atom() | duration(). +-type log_formatter() :: text | json. +-type supervisor_reports() :: error | progress. +-type log_to() :: file | console | both. +-type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. +-type flag() :: true | false. +-type duration() :: integer(). +-type duration_s() :: integer(). +-type bytesize() :: integer(). +-type percent() :: float(). +-type file() :: string(). + +-typerefl_from_string({flag/0, emqx_schema, to_flag}). +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). +-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({percent/0, emqx_schema, to_percent}). + +% workaround: prevent being recognized as unused functions +-export([to_duration/1, to_duration_s/1, to_bytesize/1, to_flag/1, to_percent/1]). + +-behaviour(hocon_schema). + +-reflect_type([ rpc_mode/0, proto_dist/0, k8s_address_type/0, port_discovery/0 + , acl_nomatch/0, acl_deny_action/0, mqueue_default_priority/0 + , endpoint/0, listener_peer_cert_tcp/0, listener_peer_cert_ssl/0 + , mqtt_piggyback/0, deflate_opts_level/0, deflate_opts_strategy/0, context_takeover/0 + , verify/0, session_locking_strategy/0 + , shared_subscription_strategy/0, route_lock_type/0 + , log_format_depth/0, log_size/0, overload_kill_restart_after/0 + , log_formatter/0, supervisor_reports/0 + , log_to/0, log_level/0, flag/0, duration/0, duration_s/0 + , bytesize/0, percent/0, file/0 +]). + +-export([structs/0, fields/1, translations/0, translation/1]). + +structs() -> ["cluster", "node", "rpc", "log", "lager", + "acl", "mqtt", "zone", "listener", "module", "broker", + "plugins", "sysmon", "os_mon", "vm_mon", "alarm", "telemetry"]. + +fields("cluster") -> + [ {"name", fun cluster__name/1} + , {"discovery", fun cluster__discovery/1} + , {"autoclean", duration("ekka.cluster_autoclean", undefined)} + , {"autoheal", flag("ekka.cluster_autoheal", false)} + , {"static", ref("static")} + , {"mcast", ref("mcast")} + , {"proto_dist", fun cluster__proto_dist/1} + , {"dns", ref("dns")} + , {"etcd", ref("etcd")} + , {"k8s", ref("k8s")} + ]; + +fields("static") -> + [ {"seeds", fun string/1}]; + +fields("mcast") -> + [ {"addr", string(undefined, "239.192.0.1")} + , {"ports", string(undefined, "4369")} + , {"iface", string(undefined, "0.0.0.0")} + , {"ttl", integer(undefined, 255)} + , {"loop", flag(undefined, true)} + , {"sndbuf", bytesize(undefined, "16KB")} + , {"recbuf", bytesize(undefined, "16KB")} + , {"buffer", bytesize(undefined, "32KB")} + ]; + +fields("dns") -> + [ {"app", fun string/1}]; + +fields("etcd") -> + [ {"server", fun string/1} + , {"prefix", fun string/1} + , {"node_ttl", duration(undefined, "1m")} + , {"ssl", ref("ssl")} + ]; + +fields("ssl") -> + [ {"keyfile", fun string/1} + , {"certfile", fun string/1} + , {"cacertfile", fun string/1} + ]; + +fields("k8s") -> + [ {"apiserver", fun string/1} + , {"service_name", fun string/1} + , {"address_type", fun cluster__k8s__address_type/1} + , {"app_name", fun string/1} + , {"namespace", fun string/1} + , {"suffix", string(undefined, "")} + ]; + +fields("node") -> + [ {"name", fun node__name/1} + , {"ssl_dist_optfile", string("vm_args.-ssl_dist_optfile", undefined)} + , {"cookie", fun node__cookie/1} + , {"data_dir", string("emqx.data_dir", undefined)} + , {"heartbeat", flag(undefined, false)} + , {"async_threads", fun node__async_threads/1} + , {"process_limit", integer("vm_args.+P", undefined)} + , {"max_ports", fun node__max_ports/1} + , {"dist_buffer_size", fun node__dist_buffer_size/1} + , {"global_gc_interval", duration_s("emqx.global_gc_interval", undefined)} + , {"fullsweep_after", fun node__fullsweep_after/1} + , {"max_ets_tables", duration("vm_args.+e", 256000)} + , {"crash_dump", fun node__crash_dump/1} + , {"dist_net_ticktime", integer("vm_args.-kernel net_ticktime", undefined)} + , {"dist_listen_min", integer("kernel.inet_dist_listen_min", undefined)} + , {"dist_listen_max", integer("kernel.inet_dist_listen_max", undefined)} + , {"backtrace_depth", integer("emqx.backtrace_depth", 16)} + ]; + +fields("rpc") -> + [ {"mode", fun rpc__mode/1} + , {"async_batch_size", integer("gen_rpc.max_batch_size", 256)} + , {"port_discovery", fun rpc__port_discovery/1} + , {"tcp_server_port", integer("gen_rpc.tcp_server_port", 5369)} + , {"tcp_client_num", fun rpc__tcp_client_num/1} + , {"connect_timeout", duration("gen_rpc.connect_timeout", "5s")} + , {"send_timeout", duration("gen_rpc.send_timeout", "5s")} + , {"authentication_timeout", duration("gen_rpc.authentication_timeout", "5s")} + , {"call_receive_timeout", duration("gen_rpc.call_receive_timeout", "15s")} + , {"socket_keepalive_idle", duration_s("gen_rpc.socket_keepalive_idle", "7200s")} + , {"socket_keepalive_interval", duration_s("gen_rpc.socket_keepalive_interval", "75s")} + , {"socket_keepalive_count", integer("gen_rpc.socket_keepalive_count", 9)} + , {"socket_sndbuf", bytesize("gen_rpc.socket_sndbuf", "1MB")} + , {"socket_recbuf", bytesize("gen_rpc.socket_recbuf", "1MB")} + , {"socket_buffer", bytesize("gen_rpc.socket_buffer", "1MB")} + ]; + +fields("log") -> + [ {"to", fun log__to/1} + , {"level", fun log__level/1} + , {"time_offset", string(undefined, "system")} + , {"primary_log_level", fun log__primary_log_level/1} + , {"dir", string(undefined,"log")} + , {"file", fun log__file/1} + , {"chars_limit", integer(undefined, -1)} + , {"supervisor_reports", fun log__supervisor_reports/1} + , {"max_depth", fun log__max_depth/1} + , {"formatter", fun log__formatter/1} + , {"single_line", boolean(undefined, true)} + , {"rotation", ref("rotation")} + , {"size", fun log__size/1} + , {"sync_mode_qlen", integer(undefined, 100)} + , {"drop_mode_qlen", integer(undefined, 3000)} + , {"flush_qlen", integer(undefined, 8000)} + , {"overload_kill", flag(undefined, true)} + , {"overload_kill_mem_size", bytesize(undefined, "30MB")} + , {"overload_kill_qlen", integer(undefined, 20000)} + , {"overload_kill_restart_after", fun log__overload_kill_restart_after/1} + , {"burst_limit", string(undefined, "disabled")} + , {"error_logger", fun log__error_logger/1} + , {"debug", ref("additional_log_file")} + , {"info", ref("additional_log_file")} + , {"notice", ref("additional_log_file")} + , {"warning", ref("additional_log_file")} + , {"error", ref("additional_log_file")} + , {"critical", ref("additional_log_file")} + , {"alert", ref("additional_log_file")} + , {"emergency", ref("additional_log_file")} + ]; + +fields("additional_log_file") -> + [ {"file", fun string/1}]; + +fields("rotation") -> + [ {"enable", flag(undefined, true)} + , {"size", bytesize(undefined, "10MB")} + , {"count", integer(undefined, 5)} + ]; + +fields("lager") -> + [ {"handlers", string("lager.handlers", "")} + , {"crash_log", flag("lager.crash_log", false)} + ]; + +fields("acl") -> + [ {"allow_anonymous", boolean("emqx.allow_anonymous", false)} + , {"acl_nomatch", fun acl_nomatch/1} + , {"acl_file", string("emqx.acl_file", undefined)} + , {"enable_acl_cache", flag("emqx.enable_acl_cache", true)} + , {"acl_cache_ttl", duration("emqx.acl_cache_ttl", "1m")} + , {"acl_cache_max_size", fun acl_cache_max_size/1} + , {"acl_deny_action", fun acl_deny_action/1} + , {"flapping_detect_policy", string(undefined, "30,1m,5m")} + ]; + +fields("mqtt") -> + [ {"max_packet_size", fun mqtt__max_packet_size/1} + , {"max_clientid_len", integer("emqx.max_clientid_len", 65535)} + , {"max_topic_levels", integer("emqx.max_topic_levels", 0)} + , {"max_qos_allowed", fun mqtt__max_qos_allowed/1} + , {"max_topic_alias", integer("emqx.max_topic_alias", 65535)} + , {"retain_available", boolean("emqx.retain_available", true)} + , {"wildcard_subscription", boolean("emqx.wildcard_subscription", true)} + , {"shared_subscription", boolean("emqx.shared_subscription", true)} + , {"ignore_loop_deliver", boolean("emqx.ignore_loop_deliver", true)} + , {"strict_mode", boolean("emqx.strict_mode", false)} + , {"response_information", string("emqx.response_information", undefined)} + ]; + +fields("zone") -> + [ {"$name", ref("zone_settings")}]; + +fields("zone_settings") -> + [ {"idle_timeout", duration(undefined, "15s")} + , {"allow_anonymous", fun boolean/1} + , {"acl_nomatch", fun zones_acl_nomatch/1} + , {"enable_acl", flag(undefined, false)} + , {"acl_deny_action", fun zones_acl_deny_action/1} + , {"enable_ban", flag(undefined, false)} + , {"enable_stats", flag(undefined, false)} + , {"max_packet_size", fun bytesize/1} + , {"max_clientid_len", fun integer/1} + , {"max_topic_levels", fun integer/1} + , {"max_qos_allowed", fun zones_max_qos_allowed/1} + , {"max_topic_alias", fun integer/1} + , {"retain_available", fun boolean/1} + , {"wildcard_subscription", fun boolean/1} + , {"shared_subscription", fun boolean/1} + , {"server_keepalive", fun integer/1} + , {"keepalive_backoff", fun zones_keepalive_backoff/1} + , {"max_subscriptions", integer(undefined, 0)} + , {"upgrade_qos", flag(undefined, false)} + , {"max_inflight", fun zones_max_inflight/1} + , {"retry_interval", duration_s(undefined, "30s")} + , {"max_awaiting_rel", duration(undefined, 0)} + , {"await_rel_timeout", duration_s(undefined, "300s")} + , {"ignore_loop_deliver", fun boolean/1} + , {"session_expiry_interval", duration_s(undefined, "2h")} + , {"max_mqueue_len", integer(undefined, 1000)} + , {"mqueue_priorities", string(undefined, "none")} + , {"mqueue_default_priority", fun zones_mqueue_default_priority/1} + , {"mqueue_store_qos0", boolean(undefined, true)} + , {"enable_flapping_detect", flag(undefined, false)} + , {"rate_limit", ref("rate_limit")} + , {"conn_congestion", ref("conn_congestion")} + , {"quota", ref("quota")} + , {"force_gc_policy", fun string/1} + , {"force_shutdown_policy", string(undefined, "default")} + , {"mountpoint", fun string/1} + , {"use_username_as_clientid", boolean(undefined, false)} + , {"strict_mode", boolean(undefined, false)} + , {"response_information", fun string/1} + , {"bypass_auth_plugins", boolean(undefined, false)} + ]; + +fields("rate_limit") -> + [ {"conn_messages_in", fun string/1} + , {"conn_bytes_in", fun string/1} + ]; + +fields("conn_congestion") -> + [ {"alarm", flag(undefined, false)} + , {"min_alarm_sustain_duration", duration(undefined, "1m")} + ]; + +fields("quota") -> + [ {"conn_messages_routing", fun string/1} + , {"overall_messages_routing", fun string/1} + ]; + +fields("listener") -> + [ {"tcp", ref("tcp_listener")} + , {"ssl", ref("ssl_listener")} + , {"ws", ref("ws_listener")} + , {"wss", ref("wss_listener")} + ]; + +fields("tcp_listener") -> + [ {"$name", ref("tcp_listener_settings")}]; + +fields("ssl_listener") -> + [ {"$name", ref("ssl_listener_settings")}]; + +fields("ws_listener") -> + [ {"$name", ref("ws_listener_settings")}]; + +fields("wss_listener") -> + [ {"$name", ref("wss_listener_settings")}]; + +fields("listener_settings") -> + [ {"endpoint", fun listener_endpoint/1} + , {"acceptors", integer(undefined, 8)} + , {"max_connections", integer(undefined, 1024)} + , {"max_conn_rate", fun integer/1} + , {"active_n", integer(undefined, 100)} + , {"zone", fun string/1} + , {"rate_limit", fun string/1} + , {"access", ref("access")} + , {"proxy_protocol", fun flag/1} + , {"proxy_protocol_timeout", fun duration/1} + , {"backlog", integer(undefined, 1024)} + , {"send_timeout", duration(undefined, "15s")} + , {"send_timeout_close", flag(undefined, true)} + , {"recbuf", fun bytesize/1} + , {"sndbuf", fun bytesize/1} + , {"buffer", fun bytesize/1} + , {"high_watermark", bytesize(undefined, "1MB")} + , {"tune_buffer", fun flag/1} + , {"nodelay", fun boolean/1} + , {"reuseaddr", fun boolean/1} + ]; + +fields("tcp_listener_settings") -> + [ {"peer_cert_as_username", fun listener_peer_cert_as_username_tcp/1} + , {"peer_cert_as_clientid", fun listener_peer_cert_as_clientid_tcp/1} + ] ++ fields("listener_settings"); + +fields("ssl_listener_settings") -> + [ {"tls_versions", fun string/1} + , {"ciphers", fun string/1} + , {"psk_ciphers", fun string/1} + , {"handshake_timeout", duration(undefined, "15s")} + , {"depth", integer(undefined, 10)} + , {"key_password", fun string/1} + , {"dhfile", fun string/1} + , {"keyfile", fun string/1} + , {"certfile", fun string/1} + , {"cacertfile", fun string/1} + , {"verify", fun listener_verify/1} + , {"fail_if_no_peer_cert", fun boolean/1} + , {"secure_renegotiate", fun flag/1} + , {"reuse_sessions", flag(undefined, true)} + , {"honor_cipher_order", fun flag/1} + , {"peer_cert_as_username", fun listener_peer_cert_as_username_ssl/1} + , {"peer_cert_as_clientid", fun listener_peer_cert_as_clientid_ssl/1} + ] ++ fields("listener_settings"); + +fields("ws_listener_settings") -> + [ {"mqtt_path", string(undefined, "/mqtt")} + , {"fail_if_no_subprotocol", boolean(undefined, true)} + , {"supported_subprotocols", string(undefined, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} + , {"proxy_address_header", string(undefined, "X-Forwarded-For")} + , {"proxy_port_header", string(undefined, "X-Forwarded-Port")} + , {"compress", fun boolean/1} + , {"deflate_opts", ref("deflate_opts")} + , {"idle_timeout", fun duration/1} + , {"max_frame_size", fun integer/1} + , {"mqtt_piggyback", fun listener_mqtt_piggyback/1} + , {"check_origin_enable", boolean(undefined, false)} + , {"allow_origin_absence", boolean(undefined, true)} + , {"check_origins", fun string/1} + % @fixme + ] ++ lists:keydelete("high_watermark", 1, fields("tcp_listener_settings")); + +fields("wss_listener_settings") -> + % @fixme + Settings = lists:ukeymerge(1, fields("ssl_listener_settings"), fields("ws_listener_settings")), + [{K, V} || {K, V} <- Settings, + lists:all(fun(X) -> X =/= K end, ["high_watermark", "handshake_timeout", "dhfile"])]; + +fields("access") -> + [ {"$id", fun string/1}]; + +fields("deflate_opts") -> + [ {"level", fun deflate_opts_level/1} + , {"mem_level", fun deflate_opts_mem_level/1} + , {"strategy", fun deflate_opts_strategy/1} + , {"server_context_takeover", fun deflate_opts_server_context_takeover/1} + , {"client_context_takeover", fun deflate_opts_client_context_takeover/1} + , {"server_max_window_bits", fun integer/1} + , {"client_max_window_bits", fun integer/1} + ]; + +fields("module") -> + [ {"loaded_file", string("emqx.modules_loaded_file", undefined)} + , {"presence", ref("presence")} + , {"subscription", ref("subscription")} + , {"rewrite", ref("rewrite")} + ]; + +fields("presence") -> + [ {"qos", fun module_presence__qos/1}]; + +fields("subscription") -> + [ {"$id", ref("subscription_settings")} + ]; + +fields("subscription_settings") -> + [ {"topic", fun string/1} + , {"qos", fun module_subscription_qos/1} + , {"nl", fun module_subscription_nl/1} + , {"rap", fun module_subscription_rap/1} + , {"rh", fun module_subscription_rh/1} + ]; + + +fields("rewrite") -> + [ {"rule", ref("rule")} + , {"pub_rule", ref("rule")} + , {"sub_rule", ref("rule")} + ]; + +fields("rule") -> + [ {"$id", fun string/1}]; + +fields("plugins") -> + [ {"etc_dir", string("emqx.plugins_etc_dir", undefined)} + , {"loaded_file", string("emqx.plugins_loaded_file", undefined)} + , {"expand_plugins_dir", string("emqx.expand_plugins_dir", undefined)} + ]; + +fields("broker") -> + [ {"sys_interval", duration("emqx.broker_sys_interval", "1m")} + , {"sys_heartbeat", duration("emqx.broker_sys_heartbeat", "30s")} + , {"enable_session_registry", flag("emqx.enable_session_registry", true)} + , {"session_locking_strategy", fun broker__session_locking_strategy/1} + , {"shared_subscription_strategy", fun broker__shared_subscription_strategy/1} + , {"shared_dispatch_ack_enabled", boolean("emqx.shared_dispatch_ack_enabled", false)} + , {"route_batch_clean", flag("emqx.route_batch_clean", true)} + , {"perf", ref("perf")} + ]; + +fields("perf") -> + [ {"route_lock_type", fun broker__perf__route_lock_type/1} + , {"trie_compaction", boolean("emqx.trie_compaction", true)} + ]; + +fields("sysmon") -> + [ {"long_gc", duration(undefined, 0)} + , {"long_schedule", duration(undefined, 240)} + , {"large_heap", bytesize(undefined, "8MB")} + , {"busy_dist_port", boolean(undefined, true)} + , {"busy_port", boolean(undefined, false)} + ]; + +fields("os_mon") -> + [ {"cpu_check_interval", duration_s(undefined, 60)} + , {"cpu_high_watermark", percent(undefined, "80%")} + , {"cpu_low_watermark", percent(undefined, "60%")} + , {"mem_check_interval", duration_s(undefined, 60)} + , {"sysmem_high_watermark", percent(undefined, "70%")} + , {"procmem_high_watermark", percent(undefined, "5%")} + ]; + +fields("vm_mon") -> + [ {"check_interval", duration_s(undefined, 30)} + , {"process_high_watermark", percent(undefined, "80%")} + , {"process_low_watermark", percent(undefined, "60%")} + ]; + +fields("alarm") -> + [ {"actions", string(undefined, "log,publish")} + , {"size_limit", integer(undefined, 1000)} + , {"validity_period", duration_s(undefined, "24h")} + ]; + +fields("telemetry") -> + [ {"enabled", boolean(undefined, false)} + , {"url", string(undefined, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry")} + , {"report_interval", duration_s(undefined, "7d")} + ]. + + +translations() -> ["ekka", "vm_args", "gen_rpc", "kernel", "emqx"]. + +translation("ekka") -> + [ {"cluster_discovery", fun tr_cluster__discovery/1}]; + +translation("vm_args") -> + [ {"+zdbbl", fun tr_zdbbl/1} + , {"-heart", fun tr_heart/1}]; + +translation("gen_rpc") -> + [ {"tcp_client_num", fun tr_tcp_client_num/1} + , {"tcp_client_port", fun tr_tcp_client_port/1}]; + +translation("kernel") -> + [ {"logger_level", fun tr_logger_level/1} + , {"logger", fun tr_logger/1}]; + +translation("emqx") -> + [ {"flapping_detect_policy", fun tr_flapping_detect_policy/1} + , {"zones", fun tr_zones/1} + , {"listeners", fun tr_listeners/1} + , {"modules", fun tr_modules/1} + , {"sysmon", fun tr_sysmon/1} + , {"os_mon", fun tr_os_mon/1} + , {"vm_mon", fun tr_vm_mon/1} + , {"alarm", fun tr_alarm/1} + , {"telemetry", fun tr_telemetry/1} + ]. + +cluster__name(mapping) -> "ekka.cluster_name"; +cluster__name(default) -> emqxcl; +cluster__name(type) -> atom(); +cluster__name(_) -> undefined. + +cluster__discovery(default) -> manual; +cluster__discovery(type) -> atom(); +cluster__discovery(_) -> undefined. + +tr_cluster__discovery(Conf) -> + Strategy = conf_get("cluster.discovery", Conf), + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + {Strategy, Filter(options(Strategy, Conf))}. + +%% @doc The erlang distributed protocol +cluster__proto_dist(mapping) -> "ekka.proto_dist"; +cluster__proto_dist(type) -> proto_dist(); +cluster__proto_dist(default) -> inet_tcp; +cluster__proto_dist(_) -> undefined. + +cluster__k8s__address_type(type) -> k8s_address_type(); +cluster__k8s__address_type(_) -> undefined. + +%% @doc Node name +node__name(mapping) -> "vm_args.-name"; +node__name(type) -> string(); +node__name(default) -> "emqx@127.0.0.1"; +node__name(override_env) -> "NODE_NAME"; +node__name(_) -> undefined. + +%% @doc Secret cookie for distributed erlang node +node__cookie(mapping) -> "vm_args.-setcookie"; +node__cookie(default) -> "emqxsecretcookie"; +node__cookie(override_env) -> "NODE_COOKIE"; +node__cookie(X) -> string(X). + +tr_heart(Conf) -> + case conf_get("node.heartbeat", Conf) of + true -> ""; + "on" -> ""; + _ -> undefined + end. + +%% @doc More information at: http://erlang.org/doc/man/erl.html +node__async_threads(mapping) -> "vm_args.+A"; +node__async_threads(type) -> range(1, 1024); +node__async_threads(_) -> undefined. + +%% @doc The maximum number of concurrent ports/sockets.Valid range is 1024-134217727 +node__max_ports(mapping) -> "vm_args.+Q"; +node__max_ports(type) -> range(1024, 134217727); +node__max_ports(override_env) -> "MAX_PORTS"; +node__max_ports(_) -> undefined. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +node__dist_buffer_size(type) -> bytesize(); +node__dist_buffer_size(validator) -> + fun(ZDBBL) -> + case ZDBBL >= 1024 andalso ZDBBL =< 2147482624 of + true -> + ok; + false -> + {error, "must be between 1KB and 2097151KB"} + end + end; +node__dist_buffer_size(_) -> undefined. + +tr_zdbbl(Conf) -> + case conf_get("node.dist_buffer_size", Conf) of + undefined -> undefined; + X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end. + +%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 +node__fullsweep_after(mapping) -> "vm_args.-env ERL_FULLSWEEP_AFTER"; +node__fullsweep_after(type) -> non_neg_integer(); +node__fullsweep_after(default) -> 1000; +node__fullsweep_after(_) -> undefined. + +%% @doc Set the location of crash dumps +node__crash_dump(mapping) -> "vm_args.-env ERL_CRASH_DUMP"; +node__crash_dump(type) -> file(); +node__crash_dump(_) -> undefined. + +rpc__mode(mapping) -> "emqx.rpc_mode"; +rpc__mode(type) -> rpc_mode(); +rpc__mode(default) -> async; +rpc__mode(_) -> undefined. + +rpc__port_discovery(mapping) -> "gen_rpc.port_discovery"; +rpc__port_discovery(type) -> port_discovery(); +rpc__port_discovery(default) -> stateless; +rpc__port_discovery(_) -> undefined. + +rpc__tcp_client_num(type) -> range(0, 255); +rpc__tcp_client_num(default) -> 0; +rpc__tcp_client_num(_) -> undefined. + +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +tr_tcp_client_num(Conf) -> + case conf_get("rpc.tcp_client_num", Conf) of + 0 -> max(1, erlang:system_info(schedulers) div 2); + V -> V + end. + +tr_tcp_client_port(Conf) -> + conf_get("rpc.tcp_server_port", Conf). + +log__to(type) -> log_to(); +log__to(default) -> file; +log__to(_) -> undefined. + +log__level(type) -> log_level(); +log__level(default) -> warning; +log__level(_) -> undefined. + +tr_logger_level(Conf) -> conf_get("log.level", Conf). + +log__primary_log_level(mapping) -> "kernel.logger_level"; +log__primary_log_level(type) -> log_level(); +log__primary_log_level(default) -> warning; +log__primary_log_level(_) -> undefined. + +log__file(type) -> file(); +log__file(default) -> "emqx.log"; +log__file(_) -> undefined. + +log__supervisor_reports(type) -> supervisor_reports(); +log__supervisor_reports(default) -> error; +log__supervisor_reports(_) -> undefined. + +%% @doc Maximum depth in Erlang term log formattingand message queue inspection. +log__max_depth(mapping) -> "kernel.error_logger_format_depth"; +log__max_depth(type) -> log_format_depth(); +log__max_depth(default) -> 20; +log__max_depth(_) -> undefined. + +%% @doc format logs as JSON objects +log__formatter(type) -> log_formatter(); +log__formatter(default) -> text; +log__formatter(_) -> undefined. + +log__size(type) -> log_size(); +log__size(default) -> infinity; +log__size(_) -> undefined. + +% @todo convert to union +log__overload_kill_restart_after(type) -> duration(); +log__overload_kill_restart_after(default) -> "5s"; +log__overload_kill_restart_after(_) -> undefined. + +log__error_logger(mapping) -> "kernel.error_logger"; +log__error_logger(type) -> atom(); +log__error_logger(default) -> silent; +log__error_logger(_) -> undefined. + +tr_logger(Conf) -> + LogTo = conf_get("log.to", Conf), + LogLevel = conf_get("log.level", Conf), + LogType = case conf_get("log.rotation.enable", Conf) of + true -> wrap; + _ -> halt + end, + CharsLimit = case conf_get("log.chars_limit", Conf) of + -1 -> unlimited; + V -> V + end, + SingleLine = conf_get("log.single_line", Conf), + FmtName = conf_get("log.formatter", Conf), + Formatter = formatter(FmtName, CharsLimit, SingleLine), + BurstLimit = string:tokens(conf_get("log.burst_limit", Conf), ", "), + {BustLimitOn, {MaxBurstCount, TimeWindow}} = burst_limit(BurstLimit), + FileConf = fun (Filename) -> + BasicConf = + #{type => LogType, + file => filename:join(conf_get("log.dir", Conf), Filename), + max_no_files => conf_get("log.rotation.count", Conf), + sync_mode_qlen => conf_get("log.sync_mode_qlen", Conf), + drop_mode_qlen => conf_get("log.drop_mode_qlen", Conf), + flush_qlen => conf_get("log.flush_qlen", Conf), + overload_kill_enable => conf_get("log.overload_kill", Conf), + overload_kill_qlen => conf_get("log.overload_kill_qlen", Conf), + overload_kill_mem_size => conf_get("log.overload_kill_mem_size", Conf), + overload_kill_restart_after => conf_get("log.overload_kill_restart_after", Conf), + burst_limit_enable => BustLimitOn, + burst_limit_max_count => MaxBurstCount, + burst_limit_window_time => TimeWindow + }, + MaxNoBytes = case LogType of + wrap -> conf_get("log.rotation.size", Conf); + halt -> conf_get("log.size", Conf) + end, + BasicConf#{max_no_bytes => MaxNoBytes} end, + + Filters = case conf_get("log.supervisor_reports", Conf) of + error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; + progress -> [] + end, + + %% For the default logger that outputs to console + DefaultHandler = + if LogTo =:= console orelse LogTo =:= both -> + [{handler, console, logger_std_h, + #{level => LogLevel, + config => #{type => standard_io}, + formatter => Formatter, + filters => Filters + } + }]; + true -> + [{handler, default, undefined}] + end, + + %% For the file logger + FileHandler = + if LogTo =:= file orelse LogTo =:= both -> + [{handler, file, logger_disk_log_h, + #{level => LogLevel, + config => FileConf(conf_get("log.file", Conf)), + formatter => Formatter, + filesync_repeat_interval => no_repeat, + filters => Filters + }}]; + true -> [] + end, + + AdditionalLogFiles = additional_log_files(Conf), + AdditionalHandlers = + [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, + #{level => list_to_atom(Level), + config => FileConf(Filename), + formatter => Formatter, + filesync_repeat_interval => no_repeat}} + || {Level, Filename} <- AdditionalLogFiles], + + DefaultHandler ++ FileHandler ++ AdditionalHandlers. + +%% @doc ACL nomatch. +acl_nomatch(mapping) -> "emqx.acl_nomatch"; +acl_nomatch(type) -> acl_nomatch(); +acl_nomatch(default) -> deny; +acl_nomatch(_) -> undefined. + +%% @doc ACL cache size. +acl_cache_max_size(mapping) -> "emqx.acl_cache_max_size"; +acl_cache_max_size(type) -> range(1, inf); +acl_cache_max_size(default) -> 32; +acl_cache_max_size(_) -> undefined. + +%% @doc Action when acl check reject current operation +acl_deny_action(mapping) -> "emqx.acl_deny_action"; +acl_deny_action(type) -> acl_deny_action(); +acl_deny_action(default) -> ignore; +acl_deny_action(_) -> undefined. + +tr_flapping_detect_policy(Conf) -> + Policy = conf_get("acl.flapping_detect_policy", Conf), + [Threshold, Duration, Interval] = string:tokens(Policy, ", "), + ParseDuration = fun(S, Dur) -> + case cuttlefish_duration:parse(S, Dur) of + I when is_integer(I) -> I; + {error, Reason} -> error(Reason) + end end, + #{threshold => list_to_integer(Threshold), + duration => ParseDuration(Duration, ms), + banned_interval => ParseDuration(Interval, s) + }. + +%% @doc Max Packet Size Allowed, 1MB by default. +mqtt__max_packet_size(mapping) -> "emqx.max_packet_size"; +mqtt__max_packet_size(override_env) -> "MAX_PACKET_SIZE"; +mqtt__max_packet_size(type) -> bytesize(); +mqtt__max_packet_size(default) -> "1MB"; +mqtt__max_packet_size(_) -> undefined. + +%% @doc Set the Maximum QoS allowed. +mqtt__max_qos_allowed(mapping) -> "emqx.max_qos_allowed"; +mqtt__max_qos_allowed(type) -> range(0, 2); +mqtt__max_qos_allowed(default) -> 2; +mqtt__max_qos_allowed(_) -> undefined. + +zones_acl_nomatch(type) -> acl_nomatch(); +zones_acl_nomatch(_) -> undefined. + +%% @doc Action when acl check reject current operation +zones_acl_deny_action(type) -> acl_deny_action(); +zones_acl_deny_action(default) -> ignore; +zones_acl_deny_action(_) -> undefined. + +%% @doc Set the Maximum QoS allowed. +zones_max_qos_allowed(type) -> range(0, 2); +zones_max_qos_allowed(_) -> undefined. + +%% @doc Keepalive backoff +zones_keepalive_backoff(type) -> float(); +zones_keepalive_backoff(default) -> 0.75; +zones_keepalive_backoff(_) -> undefined. + +%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time.0 is equivalent to maximum allowed +zones_max_inflight(type) -> range(0, 65535); +zones_max_inflight(_) -> undefined. + +%% @doc Default priority for topics not in priority table. +zones_mqueue_default_priority(type) -> mqueue_default_priority(); +zones_mqueue_default_priority(default) -> lowest; +zones_mqueue_default_priority(_) -> undefined. + +tr_zones(Conf) -> + Names = lists:usort(keys("zone", Conf)), + lists:foldl( + fun(Name, Zones) -> + Zone = keys("zone." ++ Name, Conf), + Mapped = lists:flatten([map_zones(K, conf_get(["zone", Name, K], Conf)) || K <- Zone]), + [{list_to_atom(Name), lists:filter(fun ({K, []}) when K =:= ratelimit; K =:= quota -> false; + ({_, undefined}) -> false; + (_) -> true end, Mapped)} | Zones] + end, [], Names). + +listener_endpoint(type) -> endpoint(); +listener_endpoint(_) -> undefined. + +listener_peer_cert_as_username_tcp(type) -> listener_peer_cert_tcp(); +listener_peer_cert_as_username_tcp(_) -> undefined. + +listener_peer_cert_as_clientid_tcp(type) -> listener_peer_cert_tcp(); +listener_peer_cert_as_clientid_tcp(_) -> undefined. + +listener_peer_cert_as_username_ssl(type) -> listener_peer_cert_ssl(); +listener_peer_cert_as_username_ssl(_) -> undefined. + +listener_peer_cert_as_clientid_ssl(type) -> listener_peer_cert_ssl(); +listener_peer_cert_as_clientid_ssl(_) -> undefined. + +listener_verify(type) -> verify(); +listener_verify(_) -> undefined. + +listener_mqtt_piggyback(type) -> mqtt_piggyback(); +listener_mqtt_piggyback(default) -> multiple; +listener_mqtt_piggyback(_) -> undefined. + +deflate_opts_level(type) -> deflate_opts_level(); +deflate_opts_level(_) -> undefined. + +deflate_opts_mem_level(type) -> range(1, 9); +deflate_opts_mem_level(_) -> undefined. + +deflate_opts_strategy(type) -> deflate_opts_strategy(); +deflate_opts_strategy(_) -> undefined. + +deflate_opts_server_context_takeover(type) -> context_takeover(); +deflate_opts_server_context_takeover(_) -> undefined. + +deflate_opts_client_context_takeover(type) -> context_takeover(); +deflate_opts_client_context_takeover(_) -> undefined. + +tr_listeners(Conf) -> + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + + Atom = fun(undefined) -> undefined; + (B) when is_binary(B)-> binary_to_atom(B); + (S) when is_list(S) -> list_to_atom(S) end, + + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + + AccOpts = fun(Prefix) -> + case keys(Prefix ++ ".access", Conf) of + [] -> []; + Ids -> + [{access_rules, [Access(conf_get(Prefix ++ ".access." ++ Id, Conf)) || Id <- Ids]}] + end end, + + RateLimit = fun(undefined) -> + undefined; + (Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason} -> error(Reason) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason1} -> error(Reason1) + end, + {Limit, Duration} + end, + + CheckOrigin = fun(S) -> + Origins = string:tokens(S, ","), + [ list_to_binary(string:trim(O)) || O <- Origins] + end, + + WsOpts = fun(Prefix) -> + case conf_get(Prefix ++ ".check_origins", Conf) of + undefined -> undefined; + Rules -> lists:flatten(CheckOrigin(Rules)) + end + end, + + LisOpts = fun(Prefix) -> + Filter([{acceptors, conf_get(Prefix ++ ".acceptors", Conf)}, + {mqtt_path, conf_get(Prefix ++ ".mqtt_path", Conf)}, + {max_connections, conf_get(Prefix ++ ".max_connections", Conf)}, + {max_conn_rate, conf_get(Prefix ++ ".max_conn_rate", Conf)}, + {active_n, conf_get(Prefix ++ ".active_n", Conf)}, + {tune_buffer, conf_get(Prefix ++ ".tune_buffer", Conf)}, + {zone, Atom(conf_get(Prefix ++ ".zone", Conf))}, + {rate_limit, RateLimit(conf_get(Prefix ++ ".rate_limit", Conf))}, + {proxy_protocol, conf_get(Prefix ++ ".proxy_protocol", Conf)}, + {proxy_address_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_address_header", Conf, <<"">>)))}, + {proxy_port_header, list_to_binary(string:lowercase(conf_get(Prefix ++ ".proxy_port_header", Conf, <<"">>)))}, + {proxy_protocol_timeout, conf_get(Prefix ++ ".proxy_protocol_timeout", Conf)}, + {fail_if_no_subprotocol, conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf)}, + {supported_subprotocols, string:tokens(conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, + {peer_cert_as_username, conf_get(Prefix ++ ".peer_cert_as_username", Conf)}, + {peer_cert_as_clientid, conf_get(Prefix ++ ".peer_cert_as_clientid", Conf)}, + {compress, conf_get(Prefix ++ ".compress", Conf)}, + {idle_timeout, conf_get(Prefix ++ ".idle_timeout", Conf)}, + {max_frame_size, conf_get(Prefix ++ ".max_frame_size", Conf)}, + {mqtt_piggyback, conf_get(Prefix ++ ".mqtt_piggyback", Conf)}, + {check_origin_enable, conf_get(Prefix ++ ".check_origin_enable", Conf)}, + {allow_origin_absence, conf_get(Prefix ++ ".allow_origin_absence", Conf)}, + {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + end, + DeflateOpts = fun(Prefix) -> + Filter([{level, conf_get(Prefix ++ ".deflate_opts.level", Conf)}, + {mem_level, conf_get(Prefix ++ ".deflate_opts.mem_level", Conf)}, + {strategy, conf_get(Prefix ++ ".deflate_opts.strategy", Conf)}, + {server_context_takeover, conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf)}, + {client_context_takeover, conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf)}, + {server_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf)}, + {client_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf)}]) + end, + TcpOpts = fun(Prefix) -> + Filter([{backlog, conf_get(Prefix ++ ".backlog", Conf)}, + {send_timeout, conf_get(Prefix ++ ".send_timeout", Conf)}, + {send_timeout_close, conf_get(Prefix ++ ".send_timeout_close", Conf)}, + {recbuf, conf_get(Prefix ++ ".recbuf", Conf)}, + {sndbuf, conf_get(Prefix ++ ".sndbuf", Conf)}, + {buffer, conf_get(Prefix ++ ".buffer", Conf)}, + {high_watermark, conf_get(Prefix ++ ".high_watermark", Conf)}, + {nodelay, conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, conf_get(Prefix ++ ".reuseaddr", Conf)}]) + end, + SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, + MapPSKCiphers = fun(PSKCiphers) -> + lists:map( + fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; + ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; + ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; + ("PSK-RC4-SHA") -> {psk, rc4_128, sha} + end, PSKCiphers) + end, + SslOpts = fun(Prefix) -> + Versions = case SplitFun(conf_get(Prefix ++ ".tls_versions", Conf)) of + undefined -> undefined; + L -> [list_to_atom(V) || V <- L] + end, + TLSCiphers = conf_get(Prefix++".ciphers", Conf), + PSKCiphers = conf_get(Prefix++".psk_ciphers", Conf), + Ciphers = + case {TLSCiphers, PSKCiphers} of + {undefined, undefined} -> + cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); + {TLSCiphers, undefined} -> + SplitFun(TLSCiphers); + {undefined, PSKCiphers} -> + MapPSKCiphers(SplitFun(PSKCiphers)); + {_TLSCiphers, _PSKCiphers} -> + cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") + end, + UserLookupFun = + case PSKCiphers of + undefined -> undefined; + _ -> {fun emqx_psk:lookup/3, <<>>} + end, + Filter([{versions, Versions}, + {ciphers, Ciphers}, + {user_lookup_fun, UserLookupFun}, + {handshake_timeout, conf_get(Prefix ++ ".handshake_timeout", Conf)}, + {depth, conf_get(Prefix ++ ".depth", Conf)}, + {password, conf_get(Prefix ++ ".key_password", Conf)}, + {dhfile, conf_get(Prefix ++ ".dhfile", Conf)}, + {keyfile, conf_get(Prefix ++ ".keyfile", Conf)}, + {certfile, conf_get(Prefix ++ ".certfile", Conf)}, + {cacertfile, conf_get(Prefix ++ ".cacertfile", Conf)}, + {verify, conf_get(Prefix ++ ".verify", Conf)}, + {fail_if_no_peer_cert, conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf)}, + {secure_renegotiate, conf_get(Prefix ++ ".secure_renegotiate", Conf)}, + {reuse_sessions, conf_get(Prefix ++ ".reuse_sessions", Conf)}, + {honor_cipher_order, conf_get(Prefix ++ ".honor_cipher_order", Conf)}]) + end, + + Listen_fix = fun(IpPort) when is_list(IpPort) -> + [Ip, Port] = string:tokens(IpPort, ":"), + case inet:parse_address(Ip) of + {ok, R} -> {R, list_to_integer(Port)}; + _ -> error("failed to parse ip address") + end; + (Other) -> Other end, + + TcpListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + ListenOnN = case conf_get(Prefix ++ ".endpoint", Conf) of + undefined -> []; + ListenOn -> Listen_fix(ListenOn) + end, + [#{ proto => Atom(Type) + , name => Name + , listen_on => ListenOnN + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end, + SslListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case conf_get(Prefix ++ ".endpoint", Conf) of + undefined -> + []; + ListenOn -> + [#{ proto => Atom(Type) + , name => Name + , listen_on => Listen_fix(ListenOn) + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + , {ssl_options, SslOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end + end, + + + lists:flatten([TcpListeners("tcp", Name) || Name <- keys("listener.tcp", Conf)] + ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] + ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] + ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). + +module_presence__qos(type) -> range(0, 2); +module_presence__qos(default) -> 1; +module_presence__qos(_) -> undefined. + +module_subscription_qos(type) -> range(0, 2); +module_subscription_qos(default) -> 1; +module_subscription_qos(_) -> undefined. + +module_subscription_nl(type) -> range(0, 1); +module_subscription_nl(default) -> 0; +module_subscription_nl(_) -> undefined. + +module_subscription_rap(type) -> range(0, 1); +module_subscription_rap(default) -> 0; +module_subscription_rap(_) -> undefined. + +module_subscription_rh(type) -> range(0, 2); +module_subscription_rh(default) -> 0; +module_subscription_rh(_) -> undefined. + +tr_modules(Conf) -> + Subscriptions = fun() -> + List = keys("module.subscription", Conf), + TopicList = [{N, conf_get(["module", "subscription", N, "topic"], Conf)}|| N <- List], + [{list_to_binary(T), #{ qos => conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = keys("module.rewrite.rule", Conf), + PubRules = keys("module.rewrite.pub_rule", Conf), + SubRules = keys("module.rewrite.sub_rule", Conf), + TotalRules = + [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ + [ {["module", "rewrite", "pub", "rule", R], conf_get(["module.rewrite.pub_rule", R], Conf)} || R <- PubRules] ++ + [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.rule", R], Conf)} || R <- Rules] ++ + [ {["module", "rewrite", "sub", "rule", R], conf_get(["module.rewrite.sub_rule", R], Conf)} || R <- SubRules], + lists:map(fun({[_, "rewrite", PubOrSub, "rule", _], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + [{emqx_mod_acl_internal, [{acl_file, conf_get("acl.acl_file", Conf)}]}] + ]). + +broker__session_locking_strategy(mapping) -> "emqx.session_locking_strategy"; +broker__session_locking_strategy(type) -> session_locking_strategy(); +broker__session_locking_strategy(default) -> quorum; +broker__session_locking_strategy(_) -> undefined. + +%% @doc Shared Subscription Dispatch Strategy.randomly pick a subscriber +%% round robin alive subscribers one message after another +%% pick a random subscriber and stick to it +%% hash client ID to a group member +broker__shared_subscription_strategy(mapping) -> "emqx.shared_subscription_strategy"; +broker__shared_subscription_strategy(type) -> shared_subscription_strategy(); +broker__shared_subscription_strategy(default) -> round_robin; +broker__shared_subscription_strategy(_) -> undefined. + +%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. +%% Change this toggle only when there are many wildcard topics. +%% key: mnesia translational updates with per-key locks. recommended for single node setup. +%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. +%% global: global lock protected updates. recommended for larger cluster. +%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster +broker__perf__route_lock_type(mapping) -> "emqx.route_lock_type"; +broker__perf__route_lock_type(type) -> route_lock_type(); +broker__perf__route_lock_type(default) -> key; +broker__perf__route_lock_type(_) -> undefined. + +tr_sysmon(Conf) -> + Keys = maps:to_list(conf_get("sysmon", Conf, #{})), + [{binary_to_atom(K), maps:get(value, V)} || {K, V} <- Keys]. + +tr_os_mon(Conf) -> + [{cpu_check_interval, conf_get("os_mon.cpu_check_interval", Conf)} + , {cpu_high_watermark, conf_get("os_mon.cpu_high_watermark", Conf) * 100} + , {cpu_low_watermark, conf_get("os_mon.cpu_low_watermark", Conf) * 100} + , {mem_check_interval, conf_get("os_mon.mem_check_interval", Conf)} + , {sysmem_high_watermark, conf_get("os_mon.sysmem_high_watermark", Conf) * 100} + , {procmem_high_watermark, conf_get("os_mon.procmem_high_watermark", Conf) * 100} + ]. + +tr_vm_mon(Conf) -> + [ {check_interval, conf_get("vm_mon.check_interval", Conf)} + , {process_high_watermark, conf_get("vm_mon.process_high_watermark", Conf) * 100} + , {process_low_watermark, conf_get("vm_mon.process_low_watermark", Conf) * 100} + ]. + +tr_alarm(Conf) -> + [ {actions, [list_to_atom(Action) || Action <- string:tokens(conf_get("alarm.actions", Conf), ",")]} + , {size_limit, conf_get("alarm.size_limit", Conf)} + , {validity_period, conf_get("alarm.validity_period", Conf)} + ]. + +tr_telemetry(Conf) -> + [ {enabled, conf_get("telemetry.enabled", Conf)} + , {url, conf_get("telemetry.url", Conf)} + , {report_interval, conf_get("telemetry.report_interval", Conf)} + ]. + +%% helpers + +options(static, Conf) -> + [{seeds, [list_to_atom(S) || S <- string:tokens(conf_get("cluster.static.seeds", Conf, ""), ",")]}]; +options(mcast, Conf) -> + {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), + Ports = [list_to_integer(S) || S <- string:tokens(conf_get("cluster.mcast.ports", Conf), ",")], + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, conf_get("cluster.mcast.loop", Conf, true)}]; +options(dns, Conf) -> + [{name, conf_get("cluster.dns.name", Conf)}, + {app, conf_get("cluster.dns.app", Conf)}]; +options(etcd, Conf) -> + Namespace = "cluster.etcd.ssl", + SslOpts = fun(C) -> + Options = keys(Namespace, C), + lists:map(fun(Key) -> {list_to_atom(Key), conf_get([Namespace, Key], Conf)} end, Options) end, + [{server, string:tokens(conf_get("cluster.etcd.server", Conf), ",")}, + {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, + {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, SslOpts(Conf)}]; +options(k8s, Conf) -> + [{apiserver, conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, conf_get("cluster.k8s.service_name", Conf)}, + {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, conf_get("cluster.k8s.app_name", Conf)}, + {namespace, conf_get("cluster.k8s.namespace", Conf)}, + {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; +options(manual, _Conf) -> + []. + +formatter(json, CharsLimit, SingleLine) -> + {emqx_logger_jsonfmt, + #{chars_limit => CharsLimit, + single_line => SingleLine + }}; +formatter(text, CharsLimit, SingleLine) -> + {emqx_logger_textfmt, + #{template => + [time," [",level,"] ", + {clientid, + [{peername, + [clientid,"@",peername," "], + [clientid, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"], + chars_limit => CharsLimit, + single_line => SingleLine + }}. + +burst_limit(["disabled"]) -> + {false, {20000, 1000}}; +burst_limit([Count, Window]) -> + {true, {list_to_integer(Count), + case hocon_postprocess:duration(Window) of + Secs when is_integer(Secs) -> Secs; + _ -> error({duration, Window}) + end}}. + +%% For creating additional log files for specific log levels. +additional_log_files(Conf) -> + LogLevel = ["debug", "info", "notice", "warning", + "error", "critical", "alert", "emergency"], + additional_log_files(Conf, LogLevel, []). + +additional_log_files(_Conf, [], Acc) -> + Acc; +additional_log_files(Conf, [L | More], Acc) -> + case conf_get(["log", L, "file"], Conf) of + undefined -> additional_log_files(Conf, More, Acc); + F -> additional_log_files(Conf, More, [{L, F} | Acc]) + end. + +rate_limit(Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason1} -> error(Reason1) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason} -> error(Reason) + end, + {Limit, Duration}. + +map_zones(_, undefined) -> + {undefined, undefined}; +map_zones("force_gc_policy", Val) -> + [Count, Bytes] = string:tokens(Val, "| "), + GcPolicy = case cuttlefish_bytesize:parse(Bytes) of + {error, Reason} -> + error(Reason); + Bytes1 -> + #{bytes => Bytes1, + count => list_to_integer(Count)} + end, + {force_gc_policy, GcPolicy}; +map_zones("force_shutdown_policy", "default") -> + WordSize = erlang:system_info(wordsize), + {DefaultLen, DefaultSize} = + case WordSize of + 8 -> % arch_64 + {10000, cuttlefish_bytesize:parse("64MB")}; + 4 -> % arch_32 + {1000, cuttlefish_bytesize:parse("32MB")} + end, + {force_shutdown_policy, #{message_queue_len => DefaultLen, + max_heap_size => DefaultSize div WordSize + }}; +map_zones("force_shutdown_policy", Val) -> + [Len, Siz] = string:tokens(Val, "| "), + WordSize = erlang:system_info(wordsize), + MaxSiz = case WordSize of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + ShutdownPolicy = + case cuttlefish_bytesize:parse(Siz) of + {error, Reason} -> + error(Reason); + Siz1 when Siz1 > MaxSiz -> + cuttlefish:invalid(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + Siz1 -> + #{message_queue_len => list_to_integer(Len), + max_heap_size => Siz1 div WordSize} + end, + {force_shutdown_policy, ShutdownPolicy}; +map_zones("mqueue_priorities", Val) -> + case Val of + "none" -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE + _ -> + MqueuePriorities = lists:foldl(fun(T, Acc) -> + %% NOTE: space in "= " is intended + [Topic, Prio] = string:tokens(T, "= "), + P = list_to_integer(Prio), + (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), + maps:put(iolist_to_binary(Topic), P, Acc) + end, #{}, string:tokens(Val, ",")), + {mqueue_priorities, MqueuePriorities} + end; +map_zones("mountpoint", Val) -> + {mountpoint, iolist_to_binary(Val)}; +map_zones("response_information", Val) -> + {response_information, iolist_to_binary(Val)}; +map_zones("rate_limit", Conf) -> + Messages = case conf_get("conn_messages_in", #{value => Conf}) of + undefined -> + []; + M -> + [{conn_messages_in, rate_limit(M)}] + end, + Bytes = case conf_get("conn_bytes_in", #{value => Conf}) of + undefined -> + []; + B -> + [{conn_bytes_in, rate_limit(B)}] + end, + {ratelimit, Messages ++ Bytes}; +map_zones("conn_congestion", Conf) -> + Alarm = case conf_get("alarm", #{value => Conf}) of + undefined -> + []; + A -> + [{conn_congestion_alarm_enabled, A}] + end, + MinAlarm = case conf_get("min_alarm_sustain_duration", #{value => Conf}) of + undefined -> + []; + M -> + [{conn_congestion_min_alarm_sustain_duration, M}] + end, + Alarm ++ MinAlarm; +map_zones("quota", Conf) -> + Conn = case conf_get("conn_messages_routing", #{value => Conf}) of + undefined -> + []; + C -> + [{conn_messages_routing, rate_limit(C)}] + end, + Overall = case conf_get("overall_messages_routing", #{value => Conf}) of + undefined -> + []; + O -> + [{overall_messages_routing, rate_limit(O)}] + end, + {quota, Conn ++ Overall}; +map_zones(Opt, Val) -> + {list_to_atom(Opt), Val}. + + +%% utils + +-spec(conf_get(string() | [string()], hocon:config()) -> term()). +conf_get(Key, Conf) -> + V = hocon_schema:deep_get(Key, Conf, value), + case is_binary(V) of + true -> + binary_to_list(V); + false -> + V + end. + +conf_get(Key, Conf, Default) -> + V = hocon_schema:deep_get(Key, Conf, value, Default), + case is_binary(V) of + true -> + binary_to_list(V); + false -> + V + end. + +%% @private return a list of keys in a parent field +-spec(keys(string(), hocon:config()) -> [string()]). +keys(Parent, Conf) -> + [binary_to_list(B) || B <- maps:keys(conf_get(Parent, Conf, #{}))]. + +%% types + +duration(type) -> duration(); +duration(_) -> undefined. + +flag(type) -> flag(); +flag(_) -> undefined. + +string(type) -> string(); +string(_) -> undefined. + +integer(type) -> integer(); +integer(_) -> undefined. + +bytesize(type) -> bytesize(); +bytesize(_) -> undefined. + +boolean(type) -> boolean(); +boolean(_) -> undefined. + +duration(M, D) -> + fun (type) -> duration(); (mapping) -> M; (default) -> D; (_) -> undefined end. +duration_s(M, D) -> + fun (type) -> duration_s(); (mapping) -> M; (default) -> D; (_) -> undefined end. +flag(M, D) -> + fun (type) -> flag(); (mapping) -> M; (default) -> D; (_) -> undefined end. +string(M, D) -> + fun (type) -> string(); (mapping) -> M; (default) -> D; (_) -> undefined end. +integer(M, D) -> + fun (type) -> integer(); (mapping) -> M; (default) -> D; (_) -> undefined end. +bytesize(M, D) -> + fun (type) -> bytesize(); (mapping) -> M; (default) -> D; (_) -> undefined end. +boolean(M, D) -> + fun (type) -> boolean(); (mapping) -> M; (default) -> D; (_) -> undefined end. +percent(M, D) -> + fun (type) -> percent(); (mapping) -> M; (default) -> D; (_) -> undefined end. + +ref(Field) -> + fun (type) -> Field; (_) -> undefined end. + +to_flag(Str) -> + {ok, hocon_postprocess:onoff(Str)}. + +to_duration(Str) -> + {ok, hocon_postprocess:duration(Str)}. + +to_duration_s(Str) -> + {ok, hocon_postprocess:duration(Str) div 1000}. + +to_bytesize(Str) -> + {ok, hocon_postprocess:bytesize(Str)}. + +to_percent(Str) -> + {ok, hocon_postprocess:percent(Str)}. diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl index 673cc80e9..57319ac54 100644 --- a/test/emqx_listeners_SUITE.erl +++ b/test/emqx_listeners_SUITE.erl @@ -65,10 +65,9 @@ mustache_vars() -> ]. generate_config() -> - Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), ConfFile = render_config_file(), - {ok, Conf} = hocon:load(ConfFile, #{format => proplists}), - cuttlefish_generator:map(Schema, Conf). + {ok, Conf} = hocon:load(ConfFile, #{format => richmap}), + hocon_schema:generate(emqx_schema, Conf). set_app_env({App, Lists}) -> lists:foreach(fun({acl_file, _Var}) -> From 5546d24b6db8f6703b841070a70758557daf2166 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 28 May 2021 14:12:03 +0200 Subject: [PATCH 47/58] refactor: always use hocon to generate config previously we supports skipping config generation if USE_CUTTLEFISH not set to true. it has never been tested (at least not in automated tests) now it's time to drop it. --- bin/emqx | 88 ++++++++++++++++++++++++-------------------------------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/bin/emqx b/bin/emqx index 1a85f6fbb..1487b8940 100755 --- a/bin/emqx +++ b/bin/emqx @@ -123,9 +123,6 @@ fi # Echo to stderr on errors echoerr() { echo "$@" 1>&2; } -# By default, use hocon to generate app.config and vm.args -HOCON="${USE_HOCON:-yes}" - SED_REPLACE="sed -i " case $(sed --help 2>&1) in *GNU*) SED_REPLACE="sed -i ";; @@ -202,55 +199,46 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - if [ "$HOCON" != "yes" ]; then - # Note: we have added a parameter '-vm_args' to this. It - # appears redundant but it is not! the erlang vm allows us to - # access all arguments to the erl command EXCEPT '-args_file', - # so in order to get access to this file location from within - # the vm, we need to pass it in twice. - CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args " - else - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - - set +e - # shellcheck disable=SC2086 - HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" - echo $HOCON_OUTPUT - # shellcheck disable=SC2181 - RESULT=$? - set -e - if [ $RESULT -gt 0 ]; then - echo "$HOCON_OUTPUT" - exit $RESULT - fi - # print override from environment variables (EMQX_*) - echo "$HOCON_OUTPUT" | sed -e '$d' - CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1) - - ## Merge hocon generated *.args into the vm.args - HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') - TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" - cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" - echo "" >> "$TMP_ARG_FILE" - echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" - sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do - ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') - ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') - TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') - if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then - if [ -n "$TMP_ARG_VALUE" ]; then - sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" - else - echo "$ARG_LINE" >> "$TMP_ARG_FILE" - fi - fi - done - mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE" + EMQX_LICENSE_CONF_OPTION="" + if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then + EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" fi + set +e + # shellcheck disable=SC2086 + HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + echo $HOCON_OUTPUT + # shellcheck disable=SC2181 + RESULT=$? + set -e + if [ $RESULT -gt 0 ]; then + echo "$HOCON_OUTPUT" + exit $RESULT + fi + # print override from environment variables (EMQX_*) + echo "$HOCON_OUTPUT" | sed -e '$d' + CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1) + + ## Merge hocon generated *.args into the vm.args + HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}') + TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp" + cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE" + echo "" >> "$TMP_ARG_FILE" + echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE" + sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do + ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}') + ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}') + TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}') + if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then + if [ -n "$TMP_ARG_VALUE" ]; then + sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE" + else + echo "$ARG_LINE" >> "$TMP_ARG_FILE" + fi + fi + done + mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE" + # shellcheck disable=SC2086 if ! relx_nodetool chkconfig $CONFIG_ARGS; then echoerr "Error reading $CONFIG_ARGS" From e12a629498436150f41f599d8f4217bd53a72c06 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Sat, 29 May 2021 19:19:46 +0900 Subject: [PATCH 48/58] feat(emqx_schema): add ssl field generator --- src/emqx_schema.erl | 1414 ++++++++++++++++++------------------------- 1 file changed, 597 insertions(+), 817 deletions(-) diff --git a/src/emqx_schema.erl b/src/emqx_schema.erl index c2a161d83..5a0cc210d 100644 --- a/src/emqx_schema.erl +++ b/src/emqx_schema.erl @@ -2,30 +2,6 @@ -include_lib("typerefl/include/types.hrl"). --type rpc_mode() :: sync | async. --type proto_dist() :: inet_tcp | inet6_tcp | inet_tls. --type k8s_address_type() :: ip | dns | hostname. --type port_discovery() :: manual | stateless. --type acl_nomatch() :: allow | deny. --type acl_deny_action() :: ignore | disconnect. --type mqueue_default_priority() :: highest | lowest. --type endpoint() :: integer() | string(). --type listener_peer_cert_tcp() :: cn | tmp. % @todo fix --type listener_peer_cert_ssl() :: cn | dn | crt | pem | md5. --type mqtt_piggyback() :: single | multiple. --type deflate_opts_level() :: none | default | best_compression | best_speed. --type deflate_opts_strategy() :: default | filtered | huffman_only | rle. --type context_takeover() :: takeover | no_takeover. --type verify() :: verify_peer | verify_none. --type session_locking_strategy() :: local | one | quorum | all. --type shared_subscription_strategy() :: random | round_robin | sticky | hash. --type route_lock_type() :: key | tab | global. --type log_format_depth() :: unlimited | integer(). --type log_size() :: atom() | bytesize(). --type overload_kill_restart_after() :: atom() | duration(). --type log_formatter() :: text | json. --type supervisor_reports() :: error | progress. --type log_to() :: file | console | both. -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type flag() :: true | false. -type duration() :: integer(). @@ -33,264 +9,266 @@ -type bytesize() :: integer(). -type percent() :: float(). -type file() :: string(). +-type comma_separated_list() :: list(). +-type bar_separated_list() :: list(). +-type ip_port() :: tuple(). -typerefl_from_string({flag/0, emqx_schema, to_flag}). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). -typerefl_from_string({percent/0, emqx_schema, to_percent}). +-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). +-typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}). +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). % workaround: prevent being recognized as unused functions --export([to_duration/1, to_duration_s/1, to_bytesize/1, to_flag/1, to_percent/1]). +-export([to_duration/1, to_duration_s/1, to_bytesize/1, + to_flag/1, to_percent/1, to_comma_separated_list/1, + to_bar_separated_list/1, to_ip_port/1]). -behaviour(hocon_schema). --reflect_type([ rpc_mode/0, proto_dist/0, k8s_address_type/0, port_discovery/0 - , acl_nomatch/0, acl_deny_action/0, mqueue_default_priority/0 - , endpoint/0, listener_peer_cert_tcp/0, listener_peer_cert_ssl/0 - , mqtt_piggyback/0, deflate_opts_level/0, deflate_opts_strategy/0, context_takeover/0 - , verify/0, session_locking_strategy/0 - , shared_subscription_strategy/0, route_lock_type/0 - , log_format_depth/0, log_size/0, overload_kill_restart_after/0 - , log_formatter/0, supervisor_reports/0 - , log_to/0, log_level/0, flag/0, duration/0, duration_s/0 - , bytesize/0, percent/0, file/0 -]). +-reflect_type([ log_level/0, flag/0, duration/0, duration_s/0, + bytesize/0, percent/0, file/0, + comma_separated_list/0, bar_separated_list/0, ip_port/0]). -export([structs/0, fields/1, translations/0, translation/1]). +-export([t/1, t/3, t/4, ref/1]). +-export([conf_get/2, conf_get/3, keys/2, filter/1]). +-export([ssl/2, tr_ssl/2, tr_password_hash/2]). structs() -> ["cluster", "node", "rpc", "log", "lager", - "acl", "mqtt", "zone", "listener", "module", "broker", - "plugins", "sysmon", "os_mon", "vm_mon", "alarm", "telemetry"]. + "acl", "mqtt", "zone", "listener", "module", "broker", + "plugins", "sysmon", "os_mon", "vm_mon", "alarm", "telemetry"]. fields("cluster") -> - [ {"name", fun cluster__name/1} - , {"discovery", fun cluster__discovery/1} - , {"autoclean", duration("ekka.cluster_autoclean", undefined)} - , {"autoheal", flag("ekka.cluster_autoheal", false)} - , {"static", ref("static")} - , {"mcast", ref("mcast")} - , {"proto_dist", fun cluster__proto_dist/1} - , {"dns", ref("dns")} - , {"etcd", ref("etcd")} - , {"k8s", ref("k8s")} + [ {"name", t(atom(), "ekka.cluster_name", emqxcl)} + , {"discovery", t(atom(), undefined, manual)} + , {"autoclean", t(duration(), "ekka.cluster_autoclean", undefined)} + , {"autoheal", t(flag(), "ekka.cluster_autoheal", false)} + , {"static", ref("static")} + , {"mcast", ref("mcast")} + , {"proto_dist", t(union([inet_tcp, inet6_tcp, inet_tls]), "ekka.proto_dist", inet_tcp)} + , {"dns", ref("dns")} + , {"etcd", ref("etcd")} + , {"k8s", ref("k8s")} ]; fields("static") -> - [ {"seeds", fun string/1}]; + [ {"seeds", t(comma_separated_list())}]; fields("mcast") -> - [ {"addr", string(undefined, "239.192.0.1")} - , {"ports", string(undefined, "4369")} - , {"iface", string(undefined, "0.0.0.0")} - , {"ttl", integer(undefined, 255)} - , {"loop", flag(undefined, true)} - , {"sndbuf", bytesize(undefined, "16KB")} - , {"recbuf", bytesize(undefined, "16KB")} - , {"buffer", bytesize(undefined, "32KB")} + [ {"addr", t(string(), undefined, "239.192.0.1")} + , {"ports", t(comma_separated_list(), undefined, "4369")} + , {"iface", t(string(), undefined, "0.0.0.0")} + , {"ttl", t(integer(), undefined, 255)} + , {"loop", t(flag(), undefined, true)} + , {"sndbuf", t(bytesize(), undefined, "16KB")} + , {"recbuf", t(bytesize(), undefined, "16KB")} + , {"buffer", t(bytesize(), undefined, "32KB")} ]; fields("dns") -> - [ {"app", fun string/1}]; + [ {"app", t(string())}]; fields("etcd") -> - [ {"server", fun string/1} - , {"prefix", fun string/1} - , {"node_ttl", duration(undefined, "1m")} - , {"ssl", ref("ssl")} + [ {"server", t(comma_separated_list())} + , {"prefix", t(string())} + , {"node_ttl", t(duration(), undefined, "1m")} + , {"ssl", ref("etcd_ssl")} ]; -fields("ssl") -> - [ {"keyfile", fun string/1} - , {"certfile", fun string/1} - , {"cacertfile", fun string/1} - ]; +fields("etcd_ssl") -> + ssl(undefined, #{}); fields("k8s") -> - [ {"apiserver", fun string/1} - , {"service_name", fun string/1} - , {"address_type", fun cluster__k8s__address_type/1} - , {"app_name", fun string/1} - , {"namespace", fun string/1} - , {"suffix", string(undefined, "")} + [ {"apiserver", t(string())} + , {"service_name", t(string())} + , {"address_type", t(union([ip, dns, hostname]))} + , {"app_name", t(string())} + , {"namespace", t(string())} + , {"suffix", t(string(), undefined, "")} ]; fields("node") -> - [ {"name", fun node__name/1} - , {"ssl_dist_optfile", string("vm_args.-ssl_dist_optfile", undefined)} - , {"cookie", fun node__cookie/1} - , {"data_dir", string("emqx.data_dir", undefined)} - , {"heartbeat", flag(undefined, false)} - , {"async_threads", fun node__async_threads/1} - , {"process_limit", integer("vm_args.+P", undefined)} - , {"max_ports", fun node__max_ports/1} - , {"dist_buffer_size", fun node__dist_buffer_size/1} - , {"global_gc_interval", duration_s("emqx.global_gc_interval", undefined)} - , {"fullsweep_after", fun node__fullsweep_after/1} - , {"max_ets_tables", duration("vm_args.+e", 256000)} - , {"crash_dump", fun node__crash_dump/1} - , {"dist_net_ticktime", integer("vm_args.-kernel net_ticktime", undefined)} - , {"dist_listen_min", integer("kernel.inet_dist_listen_min", undefined)} - , {"dist_listen_max", integer("kernel.inet_dist_listen_max", undefined)} - , {"backtrace_depth", integer("emqx.backtrace_depth", 16)} + [ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "NODE_NAME")} + , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)} + , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "NODE_COOKIE")} + , {"data_dir", t(string(), "emqx.data_dir", undefined)} + , {"heartbeat", t(flag(), undefined, false)} + , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)} + , {"process_limit", t(integer(), "vm_args.+P", undefined)} + , {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "MAX_PORTS")} + , {"dist_buffer_size", fun node__dist_buffer_size/1} + , {"global_gc_interval", t(duration_s(), "emqx.global_gc_interval", undefined)} + , {"fullsweep_after", t(non_neg_integer(), + "vm_args.-env ERL_FULLSWEEP_AFTER", 1000)} + , {"max_ets_tables", t(integer(), "vm_args.+e", 256000)} + , {"crash_dump", t(file(), "vm_args.-env ERL_CRASH_DUMP", undefined)} + , {"dist_net_ticktime", t(integer(), "vm_args.-kernel net_ticktime", undefined)} + , {"dist_listen_min", t(integer(), "kernel.inet_dist_listen_min", undefined)} + , {"dist_listen_max", t(integer(), "kernel.inet_dist_listen_max", undefined)} + , {"backtrace_depth", t(integer(), "emqx.backtrace_depth", 16)} ]; fields("rpc") -> - [ {"mode", fun rpc__mode/1} - , {"async_batch_size", integer("gen_rpc.max_batch_size", 256)} - , {"port_discovery", fun rpc__port_discovery/1} - , {"tcp_server_port", integer("gen_rpc.tcp_server_port", 5369)} - , {"tcp_client_num", fun rpc__tcp_client_num/1} - , {"connect_timeout", duration("gen_rpc.connect_timeout", "5s")} - , {"send_timeout", duration("gen_rpc.send_timeout", "5s")} - , {"authentication_timeout", duration("gen_rpc.authentication_timeout", "5s")} - , {"call_receive_timeout", duration("gen_rpc.call_receive_timeout", "15s")} - , {"socket_keepalive_idle", duration_s("gen_rpc.socket_keepalive_idle", "7200s")} - , {"socket_keepalive_interval", duration_s("gen_rpc.socket_keepalive_interval", "75s")} - , {"socket_keepalive_count", integer("gen_rpc.socket_keepalive_count", 9)} - , {"socket_sndbuf", bytesize("gen_rpc.socket_sndbuf", "1MB")} - , {"socket_recbuf", bytesize("gen_rpc.socket_recbuf", "1MB")} - , {"socket_buffer", bytesize("gen_rpc.socket_buffer", "1MB")} + [ {"mode", t(union(sync, async), "emqx.rpc_mode", async)} + , {"async_batch_size", t(integer(), "gen_rpc.max_batch_size", 256)} + , {"port_discovery",t(union(manual, stateless), "gen_rpc.port_discovery", stateless)} + , {"tcp_server_port", t(integer(), "gen_rpc.tcp_server_port", 5369)} + , {"tcp_client_num", t(range(0, 255), undefined, 0)} + , {"connect_timeout", t(duration(), "gen_rpc.connect_timeout", "5s")} + , {"send_timeout", t(duration(), "gen_rpc.send_timeout", "5s")} + , {"authentication_timeout", t(duration(), "gen_rpc.authentication_timeout", "5s")} + , {"call_receive_timeout", t(duration(), "gen_rpc.call_receive_timeout", "15s")} + , {"socket_keepalive_idle", t(duration_s(), "gen_rpc.socket_keepalive_idle", "7200s")} + , {"socket_keepalive_interval", t(duration_s(), "gen_rpc.socket_keepalive_interval", "75s")} + , {"socket_keepalive_count", t(integer(), "gen_rpc.socket_keepalive_count", 9)} + , {"socket_sndbuf", t(bytesize(), "gen_rpc.socket_sndbuf", "1MB")} + , {"socket_recbuf", t(bytesize(), "gen_rpc.socket_recbuf", "1MB")} + , {"socket_buffer", t(bytesize(), "gen_rpc.socket_buffer", "1MB")} ]; fields("log") -> - [ {"to", fun log__to/1} - , {"level", fun log__level/1} - , {"time_offset", string(undefined, "system")} - , {"primary_log_level", fun log__primary_log_level/1} - , {"dir", string(undefined,"log")} - , {"file", fun log__file/1} - , {"chars_limit", integer(undefined, -1)} - , {"supervisor_reports", fun log__supervisor_reports/1} - , {"max_depth", fun log__max_depth/1} - , {"formatter", fun log__formatter/1} - , {"single_line", boolean(undefined, true)} - , {"rotation", ref("rotation")} - , {"size", fun log__size/1} - , {"sync_mode_qlen", integer(undefined, 100)} - , {"drop_mode_qlen", integer(undefined, 3000)} - , {"flush_qlen", integer(undefined, 8000)} - , {"overload_kill", flag(undefined, true)} - , {"overload_kill_mem_size", bytesize(undefined, "30MB")} - , {"overload_kill_qlen", integer(undefined, 20000)} - , {"overload_kill_restart_after", fun log__overload_kill_restart_after/1} - , {"burst_limit", string(undefined, "disabled")} - , {"error_logger", fun log__error_logger/1} - , {"debug", ref("additional_log_file")} - , {"info", ref("additional_log_file")} - , {"notice", ref("additional_log_file")} - , {"warning", ref("additional_log_file")} - , {"error", ref("additional_log_file")} - , {"critical", ref("additional_log_file")} - , {"alert", ref("additional_log_file")} - , {"emergency", ref("additional_log_file")} + [ {"to", t(union([file, console, both]), undefined, file)} + , {"level", t(log_level(), undefined, warning)} + , {"time_offset", t(string(), undefined, "system")} + , {"primary_log_level", t(log_level(), undefined, warning)} + , {"dir", t(string(), undefined, "log")} + , {"file", t(file(), undefined, "emqx.log")} + , {"chars_limit", t(integer(), undefined, -1)} + , {"supervisor_reports", t(union([error, progress]), undefined, error)} + , {"max_depth", t(union([infinity, integer()]), + "kernel.error_logger_format_depth", 20)} + , {"formatter", t(union([text, json]), undefined, text)} + , {"single_line", t(boolean(), undefined, true)} + , {"rotation", ref("rotation")} + , {"size", t(union(bytesize(), infinity), undefined, infinity)} + , {"sync_mode_qlen", t(integer(), undefined, 100)} + , {"drop_mode_qlen", t(integer(), undefined, 3000)} + , {"flush_qlen", t(integer(), undefined, 8000)} + , {"overload_kill", t(flag(), undefined, true)} + , {"overload_kill_mem_size", t(bytesize(), undefined, "30MB")} + , {"overload_kill_qlen", t(integer(), undefined, 20000)} + , {"overload_kill_restart_after", t(union(duration(), infinity), undefined, "5s")} + , {"burst_limit", t(comma_separated_list(), undefined, "disabled")} + , {"error_logger", t(atom(), "kernel.error_logger", silent)} + , {"debug", ref("additional_log_file")} + , {"info", ref("additional_log_file")} + , {"notice", ref("additional_log_file")} + , {"warning", ref("additional_log_file")} + , {"error", ref("additional_log_file")} + , {"critical", ref("additional_log_file")} + , {"alert", ref("additional_log_file")} + , {"emergency", ref("additional_log_file")} ]; fields("additional_log_file") -> - [ {"file", fun string/1}]; + [ {"file", t(string())}]; fields("rotation") -> - [ {"enable", flag(undefined, true)} - , {"size", bytesize(undefined, "10MB")} - , {"count", integer(undefined, 5)} + [ {"enable", t(flag(), undefined, true)} + , {"size", t(bytesize(), undefined, "10MB")} + , {"count", t(integer(), undefined, 5)} ]; fields("lager") -> - [ {"handlers", string("lager.handlers", "")} - , {"crash_log", flag("lager.crash_log", false)} + [ {"handlers", t(string(), "lager.handlers", "")} + , {"crash_log", t(flag(), "lager.crash_log", false)} ]; fields("acl") -> - [ {"allow_anonymous", boolean("emqx.allow_anonymous", false)} - , {"acl_nomatch", fun acl_nomatch/1} - , {"acl_file", string("emqx.acl_file", undefined)} - , {"enable_acl_cache", flag("emqx.enable_acl_cache", true)} - , {"acl_cache_ttl", duration("emqx.acl_cache_ttl", "1m")} - , {"acl_cache_max_size", fun acl_cache_max_size/1} - , {"acl_deny_action", fun acl_deny_action/1} - , {"flapping_detect_policy", string(undefined, "30,1m,5m")} + [ {"allow_anonymous", t(boolean(), "emqx.allow_anonymous", false)} + , {"acl_nomatch", t(union(allow, deny), "emqx.acl_nomatch", deny)} + , {"acl_file", t(string(), "emqx.acl_file", undefined)} + , {"enable_acl_cache", t(flag(), "emqx.enable_acl_cache", true)} + , {"acl_cache_ttl", t(duration(), "emqx.acl_cache_ttl", "1m")} + , {"acl_cache_max_size", t(range(1, inf), "emqx.acl_cache_max_size", 32)} + , {"acl_deny_action", t(union(ignore, disconnect), "emqx.acl_deny_action", ignore)} + , {"flapping_detect_policy", t(comma_separated_list(), undefined, "30,1m,5m")} ]; fields("mqtt") -> - [ {"max_packet_size", fun mqtt__max_packet_size/1} - , {"max_clientid_len", integer("emqx.max_clientid_len", 65535)} - , {"max_topic_levels", integer("emqx.max_topic_levels", 0)} - , {"max_qos_allowed", fun mqtt__max_qos_allowed/1} - , {"max_topic_alias", integer("emqx.max_topic_alias", 65535)} - , {"retain_available", boolean("emqx.retain_available", true)} - , {"wildcard_subscription", boolean("emqx.wildcard_subscription", true)} - , {"shared_subscription", boolean("emqx.shared_subscription", true)} - , {"ignore_loop_deliver", boolean("emqx.ignore_loop_deliver", true)} - , {"strict_mode", boolean("emqx.strict_mode", false)} - , {"response_information", string("emqx.response_information", undefined)} + [ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "MAX_PACKET_SIZE")} + , {"max_clientid_len", t(integer(), "emqx.max_clientid_len", 65535)} + , {"max_topic_levels", t(integer(), "emqx.max_topic_levels", 0)} + , {"max_qos_allowed", t(range(0, 2), "emqx.max_qos_allowed", 2)} + , {"max_topic_alias", t(integer(), "emqx.max_topic_alias", 65535)} + , {"retain_available", t(boolean(), "emqx.retain_available", true)} + , {"wildcard_subscription", t(boolean(), "emqx.wildcard_subscription", true)} + , {"shared_subscription", t(boolean(), "emqx.shared_subscription", true)} + , {"ignore_loop_deliver", t(boolean(), "emqx.ignore_loop_deliver", true)} + , {"strict_mode", t(boolean(), "emqx.strict_mode", false)} + , {"response_information", t(string(), "emqx.response_information", undefined)} ]; fields("zone") -> [ {"$name", ref("zone_settings")}]; fields("zone_settings") -> - [ {"idle_timeout", duration(undefined, "15s")} - , {"allow_anonymous", fun boolean/1} - , {"acl_nomatch", fun zones_acl_nomatch/1} - , {"enable_acl", flag(undefined, false)} - , {"acl_deny_action", fun zones_acl_deny_action/1} - , {"enable_ban", flag(undefined, false)} - , {"enable_stats", flag(undefined, false)} - , {"max_packet_size", fun bytesize/1} - , {"max_clientid_len", fun integer/1} - , {"max_topic_levels", fun integer/1} - , {"max_qos_allowed", fun zones_max_qos_allowed/1} - , {"max_topic_alias", fun integer/1} - , {"retain_available", fun boolean/1} - , {"wildcard_subscription", fun boolean/1} - , {"shared_subscription", fun boolean/1} - , {"server_keepalive", fun integer/1} - , {"keepalive_backoff", fun zones_keepalive_backoff/1} - , {"max_subscriptions", integer(undefined, 0)} - , {"upgrade_qos", flag(undefined, false)} - , {"max_inflight", fun zones_max_inflight/1} - , {"retry_interval", duration_s(undefined, "30s")} - , {"max_awaiting_rel", duration(undefined, 0)} - , {"await_rel_timeout", duration_s(undefined, "300s")} - , {"ignore_loop_deliver", fun boolean/1} - , {"session_expiry_interval", duration_s(undefined, "2h")} - , {"max_mqueue_len", integer(undefined, 1000)} - , {"mqueue_priorities", string(undefined, "none")} - , {"mqueue_default_priority", fun zones_mqueue_default_priority/1} - , {"mqueue_store_qos0", boolean(undefined, true)} - , {"enable_flapping_detect", flag(undefined, false)} - , {"rate_limit", ref("rate_limit")} - , {"conn_congestion", ref("conn_congestion")} - , {"quota", ref("quota")} - , {"force_gc_policy", fun string/1} - , {"force_shutdown_policy", string(undefined, "default")} - , {"mountpoint", fun string/1} - , {"use_username_as_clientid", boolean(undefined, false)} - , {"strict_mode", boolean(undefined, false)} - , {"response_information", fun string/1} - , {"bypass_auth_plugins", boolean(undefined, false)} + [ {"idle_timeout", t(duration(), undefined, "15s")} + , {"allow_anonymous", t(boolean())} + , {"acl_nomatch", t(union(allow, deny))} + , {"enable_acl", t(flag(), undefined, false)} + , {"acl_deny_action", t(union(ignore, disconnect), undefined, ignore)} + , {"enable_ban", t(flag(), undefined, false)} + , {"enable_stats", t(flag(), undefined, false)} + , {"max_packet_size", t(bytesize())} + , {"max_clientid_len", t(integer())} + , {"max_topic_levels", t(integer())} + , {"max_qos_allowed", t(range(0, 2))} + , {"max_topic_alias", t(integer())} + , {"retain_available", t(boolean())} + , {"wildcard_subscription", t(boolean())} + , {"shared_subscription", t(boolean())} + , {"server_keepalive", t(integer())} + , {"keepalive_backoff", t(float(), undefined, 0.75)} + , {"max_subscriptions", t(integer(), undefined, 0)} + , {"upgrade_qos", t(flag(), undefined, false)} + , {"max_inflight", t(range(0, 65535))} + , {"retry_interval", t(duration_s(), undefined, "30s")} + , {"max_awaiting_rel", t(duration(), undefined, 0)} + , {"await_rel_timeout", t(duration_s(), undefined, "300s")} + , {"ignore_loop_deliver", t(boolean())} + , {"session_expiry_interval", t(duration_s(), undefined, "2h")} + , {"max_mqueue_len", t(integer(), undefined, 1000)} + , {"mqueue_priorities", t(comma_separated_list(), undefined, "none")} + , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} + , {"mqueue_store_qos0", t(boolean(), undefined, true)} + , {"enable_flapping_detect", t(flag(), undefined, false)} + , {"rate_limit", ref("rate_limit")} + , {"conn_congestion", ref("conn_congestion")} + , {"quota", ref("quota")} + , {"force_gc_policy", t(bar_separated_list())} + , {"force_shutdown_policy", t(bar_separated_list(), undefined, "default")} + , {"mountpoint", t(string())} + , {"use_username_as_clientid", t(boolean(), undefined, false)} + , {"strict_mode", t(boolean(), undefined, false)} + , {"response_information", t(string())} + , {"bypass_auth_plugins", t(boolean(), undefined, false)} ]; fields("rate_limit") -> - [ {"conn_messages_in", fun string/1} - , {"conn_bytes_in", fun string/1} + [ {"conn_messages_in", t(comma_separated_list())} + , {"conn_bytes_in", t(comma_separated_list())} ]; fields("conn_congestion") -> - [ {"alarm", flag(undefined, false)} - , {"min_alarm_sustain_duration", duration(undefined, "1m")} + [ {"alarm", t(flag(), undefined, false)} + , {"min_alarm_sustain_duration", t(duration(), undefined, "1m")} ]; fields("quota") -> - [ {"conn_messages_routing", fun string/1} - , {"overall_messages_routing", fun string/1} + [ {"conn_messages_routing", t(comma_separated_list())} + , {"overall_messages_routing", t(comma_separated_list())} ]; fields("listener") -> [ {"tcp", ref("tcp_listener")} - , {"ssl", ref("ssl_listener")} - , {"ws", ref("ws_listener")} - , {"wss", ref("wss_listener")} + , {"ssl", ref("ssl_listener")} + , {"ws", ref("ws_listener")} + , {"wss", ref("wss_listener")} ]; fields("tcp_listener") -> @@ -306,176 +284,166 @@ fields("wss_listener") -> [ {"$name", ref("wss_listener_settings")}]; fields("listener_settings") -> - [ {"endpoint", fun listener_endpoint/1} - , {"acceptors", integer(undefined, 8)} - , {"max_connections", integer(undefined, 1024)} - , {"max_conn_rate", fun integer/1} - , {"active_n", integer(undefined, 100)} - , {"zone", fun string/1} - , {"rate_limit", fun string/1} - , {"access", ref("access")} - , {"proxy_protocol", fun flag/1} - , {"proxy_protocol_timeout", fun duration/1} - , {"backlog", integer(undefined, 1024)} - , {"send_timeout", duration(undefined, "15s")} - , {"send_timeout_close", flag(undefined, true)} - , {"recbuf", fun bytesize/1} - , {"sndbuf", fun bytesize/1} - , {"buffer", fun bytesize/1} - , {"high_watermark", bytesize(undefined, "1MB")} - , {"tune_buffer", fun flag/1} - , {"nodelay", fun boolean/1} - , {"reuseaddr", fun boolean/1} + [ {"endpoint", t(union(ip_port(), integer()))} + , {"acceptors", t(integer(), undefined, 8)} + , {"max_connections", t(integer(), undefined, 1024)} + , {"max_conn_rate", t(integer())} + , {"active_n", t(integer(), undefined, 100)} + , {"zone", t(string())} + , {"rate_limit", t(comma_separated_list())} + , {"access", ref("access")} + , {"proxy_protocol", t(flag())} + , {"proxy_protocol_timeout", t(duration())} + , {"backlog", t(integer(), undefined, 1024)} + , {"send_timeout", t(duration(), undefined, "15s")} + , {"send_timeout_close", t(flag(), undefined, true)} + , {"recbuf", t(bytesize())} + , {"sndbuf", t(bytesize())} + , {"buffer", t(bytesize())} + , {"high_watermark", t(bytesize(), undefined, "1MB")} + , {"tune_buffer", t(flag())} + , {"nodelay", t(boolean())} + , {"reuseaddr", t(boolean())} ]; fields("tcp_listener_settings") -> - [ {"peer_cert_as_username", fun listener_peer_cert_as_username_tcp/1} - , {"peer_cert_as_clientid", fun listener_peer_cert_as_clientid_tcp/1} + [ {"peer_cert_as_username", t(cn)} + , {"peer_cert_as_clientid", t(cn)} ] ++ fields("listener_settings"); fields("ssl_listener_settings") -> - [ {"tls_versions", fun string/1} - , {"ciphers", fun string/1} - , {"psk_ciphers", fun string/1} - , {"handshake_timeout", duration(undefined, "15s")} - , {"depth", integer(undefined, 10)} - , {"key_password", fun string/1} - , {"dhfile", fun string/1} - , {"keyfile", fun string/1} - , {"certfile", fun string/1} - , {"cacertfile", fun string/1} - , {"verify", fun listener_verify/1} - , {"fail_if_no_peer_cert", fun boolean/1} - , {"secure_renegotiate", fun flag/1} - , {"reuse_sessions", flag(undefined, true)} - , {"honor_cipher_order", fun flag/1} - , {"peer_cert_as_username", fun listener_peer_cert_as_username_ssl/1} - , {"peer_cert_as_clientid", fun listener_peer_cert_as_clientid_ssl/1} - ] ++ fields("listener_settings"); + [ {"peer_cert_as_username", t(union([cn, dn, crt, pem, md5]))} + , {"peer_cert_as_clientid", t(union([cn, dn, crt, pem, md5]))} + ] ++ + ssl(undefined, #{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true}) ++ fields("listener_settings"); fields("ws_listener_settings") -> - [ {"mqtt_path", string(undefined, "/mqtt")} - , {"fail_if_no_subprotocol", boolean(undefined, true)} - , {"supported_subprotocols", string(undefined, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} - , {"proxy_address_header", string(undefined, "X-Forwarded-For")} - , {"proxy_port_header", string(undefined, "X-Forwarded-Port")} - , {"compress", fun boolean/1} - , {"deflate_opts", ref("deflate_opts")} - , {"idle_timeout", fun duration/1} - , {"max_frame_size", fun integer/1} - , {"mqtt_piggyback", fun listener_mqtt_piggyback/1} - , {"check_origin_enable", boolean(undefined, false)} - , {"allow_origin_absence", boolean(undefined, true)} - , {"check_origins", fun string/1} + [ {"mqtt_path", t(string(), undefined, "/mqtt")} + , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} + , {"supported_subprotocols", t(string(), undefined, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} + , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} + , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} + , {"compress", t(boolean())} + , {"deflate_opts", ref("deflate_opts")} + , {"idle_timeout", t(duration())} + , {"max_frame_size", t(integer())} + , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} + , {"check_origin_enable", t(boolean(), undefined, false)} + , {"allow_origin_absence", t(boolean(), undefined, true)} + , {"check_origins", t(comma_separated_list())} % @fixme ] ++ lists:keydelete("high_watermark", 1, fields("tcp_listener_settings")); fields("wss_listener_settings") -> % @fixme - Settings = lists:ukeymerge(1, fields("ssl_listener_settings"), fields("ws_listener_settings")), - [{K, V} || {K, V} <- Settings, - lists:all(fun(X) -> X =/= K end, ["high_watermark", "handshake_timeout", "dhfile"])]; + Ssl = ssl(undefined, #{depth => 10 + , reuse_sessions => true}) ++ fields("listener_settings"), + Settings = lists:ukeymerge(1, Ssl, fields("ws_listener_settings")), + lists:keydelete("high_watermark", 1, Settings); fields("access") -> - [ {"$id", fun string/1}]; + [ {"$id", t(string(), undefined, undefined)}]; fields("deflate_opts") -> - [ {"level", fun deflate_opts_level/1} - , {"mem_level", fun deflate_opts_mem_level/1} - , {"strategy", fun deflate_opts_strategy/1} - , {"server_context_takeover", fun deflate_opts_server_context_takeover/1} - , {"client_context_takeover", fun deflate_opts_client_context_takeover/1} - , {"server_max_window_bits", fun integer/1} - , {"client_max_window_bits", fun integer/1} + [ {"level", t(union([none, default, best_compression, best_speed]))} + , {"mem_level", t(range(1, 9))} + , {"strategy", t(union([default, filtered, huffman_only, rle]))} + , {"server_context_takeover", t(union(takeover, no_takeover))} + , {"client_context_takeover", t(union(takeover, no_takeover))} + , {"server_max_window_bits", t(integer())} + , {"client_max_window_bits", t(integer())} ]; fields("module") -> - [ {"loaded_file", string("emqx.modules_loaded_file", undefined)} - , {"presence", ref("presence")} - , {"subscription", ref("subscription")} - , {"rewrite", ref("rewrite")} + [ {"loaded_file", t(string(), "emqx.modules_loaded_file", undefined)} + , {"presence", ref("presence")} + , {"subscription", ref("subscription")} + , {"rewrite", ref("rewrite")} ]; fields("presence") -> - [ {"qos", fun module_presence__qos/1}]; + [ {"qos", t(range(0, 2), undefined, 1)}]; fields("subscription") -> - [ {"$id", ref("subscription_settings")} - ]; + [ {"$id", ref("subscription_settings")}]; fields("subscription_settings") -> - [ {"topic", fun string/1} - , {"qos", fun module_subscription_qos/1} - , {"nl", fun module_subscription_nl/1} - , {"rap", fun module_subscription_rap/1} - , {"rh", fun module_subscription_rh/1} + [ {"topic", t(string())} + , {"qos", t(range(0, 2), undefined, 1)} + , {"nl", t(range(0, 1), undefined, 0)} + , {"rap", t(range(0, 1), undefined, 0)} + , {"rh", t(range(0, 2), undefined, 0)} ]; fields("rewrite") -> [ {"rule", ref("rule")} - , {"pub_rule", ref("rule")} - , {"sub_rule", ref("rule")} + , {"pub_rule", ref("rule")} + , {"sub_rule", ref("rule")} ]; fields("rule") -> - [ {"$id", fun string/1}]; + [ {"$id", t(string())}]; fields("plugins") -> - [ {"etc_dir", string("emqx.plugins_etc_dir", undefined)} - , {"loaded_file", string("emqx.plugins_loaded_file", undefined)} - , {"expand_plugins_dir", string("emqx.expand_plugins_dir", undefined)} + [ {"etc_dir", t(string(), "emqx.plugins_etc_dir", undefined)} + , {"loaded_file", t(string(), "emqx.plugins_loaded_file", undefined)} + , {"expand_plugins_dir", t(string(), "emqx.expand_plugins_dir", undefined)} ]; fields("broker") -> - [ {"sys_interval", duration("emqx.broker_sys_interval", "1m")} - , {"sys_heartbeat", duration("emqx.broker_sys_heartbeat", "30s")} - , {"enable_session_registry", flag("emqx.enable_session_registry", true)} - , {"session_locking_strategy", fun broker__session_locking_strategy/1} - , {"shared_subscription_strategy", fun broker__shared_subscription_strategy/1} - , {"shared_dispatch_ack_enabled", boolean("emqx.shared_dispatch_ack_enabled", false)} - , {"route_batch_clean", flag("emqx.route_batch_clean", true)} - , {"perf", ref("perf")} + [ {"sys_interval", t(duration(), "emqx.broker_sys_interval", "1m")} + , {"sys_heartbeat", t(duration(), "emqx.broker_sys_heartbeat", "30s")} + , {"enable_session_registry", t(flag(), "emqx.enable_session_registry", true)} + , {"session_locking_strategy", t(union([local, leader, quorum, all]), + "emqx.session_locking_strategy", quorum)} + , {"shared_subscription_strategy", t(union(random, round_robin), + "emqx.shared_subscription_strategy", round_robin)} + , {"shared_dispatch_ack_enabled", t(boolean(), "emqx.shared_dispatch_ack_enabled", false)} + , {"route_batch_clean", t(flag(), "emqx.route_batch_clean", true)} + , {"perf", ref("perf")} ]; fields("perf") -> - [ {"route_lock_type", fun broker__perf__route_lock_type/1} - , {"trie_compaction", boolean("emqx.trie_compaction", true)} + [ {"route_lock_type", t(union([key, tab, global]), "emqx.route_lock_type", key)} + , {"trie_compaction", t(boolean(), "emqx.trie_compaction", true)} ]; fields("sysmon") -> - [ {"long_gc", duration(undefined, 0)} - , {"long_schedule", duration(undefined, 240)} - , {"large_heap", bytesize(undefined, "8MB")} - , {"busy_dist_port", boolean(undefined, true)} - , {"busy_port", boolean(undefined, false)} + [ {"long_gc", t(duration(), undefined, 0)} + , {"long_schedule", t(duration(), undefined, 240)} + , {"large_heap", t(bytesize(), undefined, "8MB")} + , {"busy_dist_port", t(boolean(), undefined, true)} + , {"busy_port", t(boolean(), undefined, false)} ]; fields("os_mon") -> - [ {"cpu_check_interval", duration_s(undefined, 60)} - , {"cpu_high_watermark", percent(undefined, "80%")} - , {"cpu_low_watermark", percent(undefined, "60%")} - , {"mem_check_interval", duration_s(undefined, 60)} - , {"sysmem_high_watermark", percent(undefined, "70%")} - , {"procmem_high_watermark", percent(undefined, "5%")} + [ {"cpu_check_interval", t(duration_s(), undefined, 60)} + , {"cpu_high_watermark", t(percent(), undefined, "80%")} + , {"cpu_low_watermark", t(percent(), undefined, "60%")} + , {"mem_check_interval", t(duration_s(), undefined, 60)} + , {"sysmem_high_watermark", t(percent(), undefined, "70%")} + , {"procmem_high_watermark", t(percent(), undefined, "5%")} ]; fields("vm_mon") -> - [ {"check_interval", duration_s(undefined, 30)} - , {"process_high_watermark", percent(undefined, "80%")} - , {"process_low_watermark", percent(undefined, "60%")} + [ {"check_interval", t(duration_s(), undefined, 30)} + , {"process_high_watermark", t(percent(), undefined, "80%")} + , {"process_low_watermark", t(percent(), undefined, "60%")} ]; fields("alarm") -> - [ {"actions", string(undefined, "log,publish")} - , {"size_limit", integer(undefined, 1000)} - , {"validity_period", duration_s(undefined, "24h")} + [ {"actions", t(comma_separated_list(), undefined, "log,publish")} + , {"size_limit", t(integer(), undefined, 1000)} + , {"validity_period", t(duration_s(), undefined, "24h")} ]; fields("telemetry") -> - [ {"enabled", boolean(undefined, false)} - , {"url", string(undefined, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry")} - , {"report_interval", duration_s(undefined, "7d")} + [ {"enabled", t(boolean(), undefined, false)} + , {"url", t(string(), undefined, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry")} + , {"report_interval", t(duration_s(), undefined, "7d")} ]. @@ -486,63 +454,31 @@ translation("ekka") -> translation("vm_args") -> [ {"+zdbbl", fun tr_zdbbl/1} - , {"-heart", fun tr_heart/1}]; + , {"-heart", fun tr_heart/1}]; translation("gen_rpc") -> [ {"tcp_client_num", fun tr_tcp_client_num/1} - , {"tcp_client_port", fun tr_tcp_client_port/1}]; + , {"tcp_client_port", fun tr_tcp_client_port/1}]; translation("kernel") -> [ {"logger_level", fun tr_logger_level/1} - , {"logger", fun tr_logger/1}]; + , {"logger", fun tr_logger/1}]; translation("emqx") -> [ {"flapping_detect_policy", fun tr_flapping_detect_policy/1} - , {"zones", fun tr_zones/1} - , {"listeners", fun tr_listeners/1} - , {"modules", fun tr_modules/1} - , {"sysmon", fun tr_sysmon/1} - , {"os_mon", fun tr_os_mon/1} - , {"vm_mon", fun tr_vm_mon/1} - , {"alarm", fun tr_alarm/1} - , {"telemetry", fun tr_telemetry/1} + , {"zones", fun tr_zones/1} + , {"listeners", fun tr_listeners/1} + , {"modules", fun tr_modules/1} + , {"sysmon", fun tr_sysmon/1} + , {"os_mon", fun tr_os_mon/1} + , {"vm_mon", fun tr_vm_mon/1} + , {"alarm", fun tr_alarm/1} + , {"telemetry", fun tr_telemetry/1} ]. -cluster__name(mapping) -> "ekka.cluster_name"; -cluster__name(default) -> emqxcl; -cluster__name(type) -> atom(); -cluster__name(_) -> undefined. - -cluster__discovery(default) -> manual; -cluster__discovery(type) -> atom(); -cluster__discovery(_) -> undefined. - tr_cluster__discovery(Conf) -> Strategy = conf_get("cluster.discovery", Conf), - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - {Strategy, Filter(options(Strategy, Conf))}. - -%% @doc The erlang distributed protocol -cluster__proto_dist(mapping) -> "ekka.proto_dist"; -cluster__proto_dist(type) -> proto_dist(); -cluster__proto_dist(default) -> inet_tcp; -cluster__proto_dist(_) -> undefined. - -cluster__k8s__address_type(type) -> k8s_address_type(); -cluster__k8s__address_type(_) -> undefined. - -%% @doc Node name -node__name(mapping) -> "vm_args.-name"; -node__name(type) -> string(); -node__name(default) -> "emqx@127.0.0.1"; -node__name(override_env) -> "NODE_NAME"; -node__name(_) -> undefined. - -%% @doc Secret cookie for distributed erlang node -node__cookie(mapping) -> "vm_args.-setcookie"; -node__cookie(default) -> "emqxsecretcookie"; -node__cookie(override_env) -> "NODE_COOKIE"; -node__cookie(X) -> string(X). + {Strategy, filter(options(Strategy, Conf))}. tr_heart(Conf) -> case conf_get("node.heartbeat", Conf) of @@ -551,17 +487,6 @@ tr_heart(Conf) -> _ -> undefined end. -%% @doc More information at: http://erlang.org/doc/man/erl.html -node__async_threads(mapping) -> "vm_args.+A"; -node__async_threads(type) -> range(1, 1024); -node__async_threads(_) -> undefined. - -%% @doc The maximum number of concurrent ports/sockets.Valid range is 1024-134217727 -node__max_ports(mapping) -> "vm_args.+Q"; -node__max_ports(type) -> range(1024, 134217727); -node__max_ports(override_env) -> "MAX_PORTS"; -node__max_ports(_) -> undefined. - %% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl node__dist_buffer_size(type) -> bytesize(); node__dist_buffer_size(validator) -> @@ -578,35 +503,10 @@ node__dist_buffer_size(_) -> undefined. tr_zdbbl(Conf) -> case conf_get("node.dist_buffer_size", Conf) of undefined -> undefined; - X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + X when is_integer(X) -> ceiling(X / 1024); %% Bytes to Kilobytes; _ -> undefined end. -%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 -node__fullsweep_after(mapping) -> "vm_args.-env ERL_FULLSWEEP_AFTER"; -node__fullsweep_after(type) -> non_neg_integer(); -node__fullsweep_after(default) -> 1000; -node__fullsweep_after(_) -> undefined. - -%% @doc Set the location of crash dumps -node__crash_dump(mapping) -> "vm_args.-env ERL_CRASH_DUMP"; -node__crash_dump(type) -> file(); -node__crash_dump(_) -> undefined. - -rpc__mode(mapping) -> "emqx.rpc_mode"; -rpc__mode(type) -> rpc_mode(); -rpc__mode(default) -> async; -rpc__mode(_) -> undefined. - -rpc__port_discovery(mapping) -> "gen_rpc.port_discovery"; -rpc__port_discovery(type) -> port_discovery(); -rpc__port_discovery(default) -> stateless; -rpc__port_discovery(_) -> undefined. - -rpc__tcp_client_num(type) -> range(0, 255); -rpc__tcp_client_num(default) -> 0; -rpc__tcp_client_num(_) -> undefined. - %% Force client to use server listening port, because we do no provide %% per-node listening port manual mapping from configs. %% i.e. all nodes in the cluster should agree to the same @@ -620,54 +520,8 @@ tr_tcp_client_num(Conf) -> tr_tcp_client_port(Conf) -> conf_get("rpc.tcp_server_port", Conf). -log__to(type) -> log_to(); -log__to(default) -> file; -log__to(_) -> undefined. - -log__level(type) -> log_level(); -log__level(default) -> warning; -log__level(_) -> undefined. - tr_logger_level(Conf) -> conf_get("log.level", Conf). -log__primary_log_level(mapping) -> "kernel.logger_level"; -log__primary_log_level(type) -> log_level(); -log__primary_log_level(default) -> warning; -log__primary_log_level(_) -> undefined. - -log__file(type) -> file(); -log__file(default) -> "emqx.log"; -log__file(_) -> undefined. - -log__supervisor_reports(type) -> supervisor_reports(); -log__supervisor_reports(default) -> error; -log__supervisor_reports(_) -> undefined. - -%% @doc Maximum depth in Erlang term log formattingand message queue inspection. -log__max_depth(mapping) -> "kernel.error_logger_format_depth"; -log__max_depth(type) -> log_format_depth(); -log__max_depth(default) -> 20; -log__max_depth(_) -> undefined. - -%% @doc format logs as JSON objects -log__formatter(type) -> log_formatter(); -log__formatter(default) -> text; -log__formatter(_) -> undefined. - -log__size(type) -> log_size(); -log__size(default) -> infinity; -log__size(_) -> undefined. - -% @todo convert to union -log__overload_kill_restart_after(type) -> duration(); -log__overload_kill_restart_after(default) -> "5s"; -log__overload_kill_restart_after(_) -> undefined. - -log__error_logger(mapping) -> "kernel.error_logger"; -log__error_logger(type) -> atom(); -log__error_logger(default) -> silent; -log__error_logger(_) -> undefined. - tr_logger(Conf) -> LogTo = conf_get("log.to", Conf), LogLevel = conf_get("log.level", Conf), @@ -682,7 +536,7 @@ tr_logger(Conf) -> SingleLine = conf_get("log.single_line", Conf), FmtName = conf_get("log.formatter", Conf), Formatter = formatter(FmtName, CharsLimit, SingleLine), - BurstLimit = string:tokens(conf_get("log.burst_limit", Conf), ", "), + BurstLimit = conf_get("log.burst_limit", Conf), {BustLimitOn, {MaxBurstCount, TimeWindow}} = burst_limit(BurstLimit), FileConf = fun (Filename) -> BasicConf = @@ -749,130 +603,33 @@ tr_logger(Conf) -> DefaultHandler ++ FileHandler ++ AdditionalHandlers. -%% @doc ACL nomatch. -acl_nomatch(mapping) -> "emqx.acl_nomatch"; -acl_nomatch(type) -> acl_nomatch(); -acl_nomatch(default) -> deny; -acl_nomatch(_) -> undefined. - -%% @doc ACL cache size. -acl_cache_max_size(mapping) -> "emqx.acl_cache_max_size"; -acl_cache_max_size(type) -> range(1, inf); -acl_cache_max_size(default) -> 32; -acl_cache_max_size(_) -> undefined. - -%% @doc Action when acl check reject current operation -acl_deny_action(mapping) -> "emqx.acl_deny_action"; -acl_deny_action(type) -> acl_deny_action(); -acl_deny_action(default) -> ignore; -acl_deny_action(_) -> undefined. - tr_flapping_detect_policy(Conf) -> - Policy = conf_get("acl.flapping_detect_policy", Conf), - [Threshold, Duration, Interval] = string:tokens(Policy, ", "), - ParseDuration = fun(S, Dur) -> - case cuttlefish_duration:parse(S, Dur) of - I when is_integer(I) -> I; - {error, Reason} -> error(Reason) + [Threshold, Duration, Interval] = conf_get("acl.flapping_detect_policy", Conf), + ParseDuration = fun(S, F) -> + case F(S) of + {ok, I} -> I; + {error, Reason} -> error({duration, Reason}) end end, #{threshold => list_to_integer(Threshold), - duration => ParseDuration(Duration, ms), - banned_interval => ParseDuration(Interval, s) + duration => ParseDuration(Duration, fun to_duration/1), + banned_interval => ParseDuration(Interval, fun to_duration_s/1) }. -%% @doc Max Packet Size Allowed, 1MB by default. -mqtt__max_packet_size(mapping) -> "emqx.max_packet_size"; -mqtt__max_packet_size(override_env) -> "MAX_PACKET_SIZE"; -mqtt__max_packet_size(type) -> bytesize(); -mqtt__max_packet_size(default) -> "1MB"; -mqtt__max_packet_size(_) -> undefined. - -%% @doc Set the Maximum QoS allowed. -mqtt__max_qos_allowed(mapping) -> "emqx.max_qos_allowed"; -mqtt__max_qos_allowed(type) -> range(0, 2); -mqtt__max_qos_allowed(default) -> 2; -mqtt__max_qos_allowed(_) -> undefined. - -zones_acl_nomatch(type) -> acl_nomatch(); -zones_acl_nomatch(_) -> undefined. - -%% @doc Action when acl check reject current operation -zones_acl_deny_action(type) -> acl_deny_action(); -zones_acl_deny_action(default) -> ignore; -zones_acl_deny_action(_) -> undefined. - -%% @doc Set the Maximum QoS allowed. -zones_max_qos_allowed(type) -> range(0, 2); -zones_max_qos_allowed(_) -> undefined. - -%% @doc Keepalive backoff -zones_keepalive_backoff(type) -> float(); -zones_keepalive_backoff(default) -> 0.75; -zones_keepalive_backoff(_) -> undefined. - -%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time.0 is equivalent to maximum allowed -zones_max_inflight(type) -> range(0, 65535); -zones_max_inflight(_) -> undefined. - -%% @doc Default priority for topics not in priority table. -zones_mqueue_default_priority(type) -> mqueue_default_priority(); -zones_mqueue_default_priority(default) -> lowest; -zones_mqueue_default_priority(_) -> undefined. - tr_zones(Conf) -> Names = lists:usort(keys("zone", Conf)), - lists:foldl( - fun(Name, Zones) -> - Zone = keys("zone." ++ Name, Conf), - Mapped = lists:flatten([map_zones(K, conf_get(["zone", Name, K], Conf)) || K <- Zone]), - [{list_to_atom(Name), lists:filter(fun ({K, []}) when K =:= ratelimit; K =:= quota -> false; - ({_, undefined}) -> false; - (_) -> true end, Mapped)} | Zones] - end, [], Names). - -listener_endpoint(type) -> endpoint(); -listener_endpoint(_) -> undefined. - -listener_peer_cert_as_username_tcp(type) -> listener_peer_cert_tcp(); -listener_peer_cert_as_username_tcp(_) -> undefined. - -listener_peer_cert_as_clientid_tcp(type) -> listener_peer_cert_tcp(); -listener_peer_cert_as_clientid_tcp(_) -> undefined. - -listener_peer_cert_as_username_ssl(type) -> listener_peer_cert_ssl(); -listener_peer_cert_as_username_ssl(_) -> undefined. - -listener_peer_cert_as_clientid_ssl(type) -> listener_peer_cert_ssl(); -listener_peer_cert_as_clientid_ssl(_) -> undefined. - -listener_verify(type) -> verify(); -listener_verify(_) -> undefined. - -listener_mqtt_piggyback(type) -> mqtt_piggyback(); -listener_mqtt_piggyback(default) -> multiple; -listener_mqtt_piggyback(_) -> undefined. - -deflate_opts_level(type) -> deflate_opts_level(); -deflate_opts_level(_) -> undefined. - -deflate_opts_mem_level(type) -> range(1, 9); -deflate_opts_mem_level(_) -> undefined. - -deflate_opts_strategy(type) -> deflate_opts_strategy(); -deflate_opts_strategy(_) -> undefined. - -deflate_opts_server_context_takeover(type) -> context_takeover(); -deflate_opts_server_context_takeover(_) -> undefined. - -deflate_opts_client_context_takeover(type) -> context_takeover(); -deflate_opts_client_context_takeover(_) -> undefined. + lists:foldl( + fun(Name, Zones) -> + Zone = keys("zone." ++ Name, Conf), + Mapped = lists:flatten([map_zones(K, conf_get(["zone", Name, K], Conf)) || K <- Zone]), + [{list_to_atom(Name), lists:filter(fun ({K, []}) when K =:= ratelimit; K =:= quota -> false; + ({_, undefined}) -> false; + (_) -> true end, Mapped)} | Zones] + end, [], Names). tr_listeners(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - Atom = fun(undefined) -> undefined; - (B) when is_binary(B)-> binary_to_atom(B); - (S) when is_list(S) -> list_to_atom(S) end, + (B) when is_binary(B)-> binary_to_atom(B); + (S) when is_list(S) -> list_to_atom(S) end, Access = fun(S) -> [A, CIDR] = string:tokens(S, " "), @@ -888,23 +645,19 @@ tr_listeners(Conf) -> RateLimit = fun(undefined) -> undefined; - (Val) -> - [L, D] = string:tokens(Val, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason} -> error(Reason) + ([L, D]) -> + Limit = case to_bytesize(L) of + {ok, I0} -> I0; + {error, R0} -> error({bytesize, R0}) end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; - {error, Reason1} -> error(Reason1) + Duration = case to_duration_s(D) of + {ok, I1} -> I1; + {error, R1} -> error({duration, R1}) end, {Limit, Duration} end, - CheckOrigin = fun(S) -> - Origins = string:tokens(S, ","), - [ list_to_binary(string:trim(O)) || O <- Origins] - end, + CheckOrigin = fun(S) -> [ list_to_binary(string:trim(O)) || O <- S] end, WsOpts = fun(Prefix) -> case conf_get(Prefix ++ ".check_origins", Conf) of @@ -914,7 +667,7 @@ tr_listeners(Conf) -> end, LisOpts = fun(Prefix) -> - Filter([{acceptors, conf_get(Prefix ++ ".acceptors", Conf)}, + filter([{acceptors, conf_get(Prefix ++ ".acceptors", Conf)}, {mqtt_path, conf_get(Prefix ++ ".mqtt_path", Conf)}, {max_connections, conf_get(Prefix ++ ".max_connections", Conf)}, {max_conn_rate, conf_get(Prefix ++ ".max_conn_rate", Conf)}, @@ -939,87 +692,40 @@ tr_listeners(Conf) -> {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) end, DeflateOpts = fun(Prefix) -> - Filter([{level, conf_get(Prefix ++ ".deflate_opts.level", Conf)}, - {mem_level, conf_get(Prefix ++ ".deflate_opts.mem_level", Conf)}, - {strategy, conf_get(Prefix ++ ".deflate_opts.strategy", Conf)}, - {server_context_takeover, conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf)}, - {client_context_takeover, conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf)}, - {server_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf)}, - {client_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf)}]) + filter([{level, conf_get(Prefix ++ ".deflate_opts.level", Conf)}, + {mem_level, conf_get(Prefix ++ ".deflate_opts.mem_level", Conf)}, + {strategy, conf_get(Prefix ++ ".deflate_opts.strategy", Conf)}, + {server_context_takeover, conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf)}, + {client_context_takeover, conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf)}, + {server_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf)}, + {client_max_windows_bits, conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf)}]) end, TcpOpts = fun(Prefix) -> - Filter([{backlog, conf_get(Prefix ++ ".backlog", Conf)}, - {send_timeout, conf_get(Prefix ++ ".send_timeout", Conf)}, - {send_timeout_close, conf_get(Prefix ++ ".send_timeout_close", Conf)}, - {recbuf, conf_get(Prefix ++ ".recbuf", Conf)}, - {sndbuf, conf_get(Prefix ++ ".sndbuf", Conf)}, - {buffer, conf_get(Prefix ++ ".buffer", Conf)}, - {high_watermark, conf_get(Prefix ++ ".high_watermark", Conf)}, - {nodelay, conf_get(Prefix ++ ".nodelay", Conf, true)}, - {reuseaddr, conf_get(Prefix ++ ".reuseaddr", Conf)}]) - end, - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - MapPSKCiphers = fun(PSKCiphers) -> - lists:map( - fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; - ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; - ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; - ("PSK-RC4-SHA") -> {psk, rc4_128, sha} - end, PSKCiphers) - end, - SslOpts = fun(Prefix) -> - Versions = case SplitFun(conf_get(Prefix ++ ".tls_versions", Conf)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - TLSCiphers = conf_get(Prefix++".ciphers", Conf), - PSKCiphers = conf_get(Prefix++".psk_ciphers", Conf), - Ciphers = - case {TLSCiphers, PSKCiphers} of - {undefined, undefined} -> - cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); - {TLSCiphers, undefined} -> - SplitFun(TLSCiphers); - {undefined, PSKCiphers} -> - MapPSKCiphers(SplitFun(PSKCiphers)); - {_TLSCiphers, _PSKCiphers} -> - cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") - end, - UserLookupFun = - case PSKCiphers of - undefined -> undefined; - _ -> {fun emqx_psk:lookup/3, <<>>} - end, - Filter([{versions, Versions}, - {ciphers, Ciphers}, - {user_lookup_fun, UserLookupFun}, - {handshake_timeout, conf_get(Prefix ++ ".handshake_timeout", Conf)}, - {depth, conf_get(Prefix ++ ".depth", Conf)}, - {password, conf_get(Prefix ++ ".key_password", Conf)}, - {dhfile, conf_get(Prefix ++ ".dhfile", Conf)}, - {keyfile, conf_get(Prefix ++ ".keyfile", Conf)}, - {certfile, conf_get(Prefix ++ ".certfile", Conf)}, - {cacertfile, conf_get(Prefix ++ ".cacertfile", Conf)}, - {verify, conf_get(Prefix ++ ".verify", Conf)}, - {fail_if_no_peer_cert, conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf)}, - {secure_renegotiate, conf_get(Prefix ++ ".secure_renegotiate", Conf)}, - {reuse_sessions, conf_get(Prefix ++ ".reuse_sessions", Conf)}, - {honor_cipher_order, conf_get(Prefix ++ ".honor_cipher_order", Conf)}]) + filter([{backlog, conf_get(Prefix ++ ".backlog", Conf)}, + {send_timeout, conf_get(Prefix ++ ".send_timeout", Conf)}, + {send_timeout_close, conf_get(Prefix ++ ".send_timeout_close", Conf)}, + {recbuf, conf_get(Prefix ++ ".recbuf", Conf)}, + {sndbuf, conf_get(Prefix ++ ".sndbuf", Conf)}, + {buffer, conf_get(Prefix ++ ".buffer", Conf)}, + {high_watermark, conf_get(Prefix ++ ".high_watermark", Conf)}, + {nodelay, conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, conf_get(Prefix ++ ".reuseaddr", Conf)}]) end, - Listen_fix = fun(IpPort) when is_list(IpPort) -> - [Ip, Port] = string:tokens(IpPort, ":"), - case inet:parse_address(Ip) of - {ok, R} -> {R, list_to_integer(Port)}; - _ -> error("failed to parse ip address") - end; - (Other) -> Other end, + SslOpts = fun(Prefix) -> + Opts = tr_ssl(Prefix, Conf), + case lists:keyfind(ciphers, 1, Opts) of + false -> + error(Prefix ++ ".ciphers or " ++ Prefix ++ ".psk_ciphers is absent"); + _ -> + Opts + end end, TcpListeners = fun(Type, Name) -> Prefix = string:join(["listener", Type, Name], "."), ListenOnN = case conf_get(Prefix ++ ".endpoint", Conf) of undefined -> []; - ListenOn -> Listen_fix(ListenOn) + ListenOn -> ListenOn end, [#{ proto => Atom(Type) , name => Name @@ -1039,7 +745,7 @@ tr_listeners(Conf) -> ListenOn -> [#{ proto => Atom(Type) , name => Name - , listen_on => Listen_fix(ListenOn) + , listen_on => ListenOn , opts => [ {deflate_options, DeflateOpts(Prefix)} , {tcp_options, TcpOpts(Prefix)} , {ssl_options, SslOpts(Prefix)} @@ -1047,43 +753,22 @@ tr_listeners(Conf) -> ] } ] - end - end, + end end, lists:flatten([TcpListeners("tcp", Name) || Name <- keys("listener.tcp", Conf)] - ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] - ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] - ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). - -module_presence__qos(type) -> range(0, 2); -module_presence__qos(default) -> 1; -module_presence__qos(_) -> undefined. - -module_subscription_qos(type) -> range(0, 2); -module_subscription_qos(default) -> 1; -module_subscription_qos(_) -> undefined. - -module_subscription_nl(type) -> range(0, 1); -module_subscription_nl(default) -> 0; -module_subscription_nl(_) -> undefined. - -module_subscription_rap(type) -> range(0, 1); -module_subscription_rap(default) -> 0; -module_subscription_rap(_) -> undefined. - -module_subscription_rh(type) -> range(0, 2); -module_subscription_rh(default) -> 0; -module_subscription_rh(_) -> undefined. + ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] + ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] + ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). tr_modules(Conf) -> Subscriptions = fun() -> List = keys("module.subscription", Conf), TopicList = [{N, conf_get(["module", "subscription", N, "topic"], Conf)}|| N <- List], [{list_to_binary(T), #{ qos => conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), - nl => conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), - rap => conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), - rh => conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + nl => conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) }} || {N, T} <- TopicList] end, Rewrites = fun() -> @@ -1109,92 +794,67 @@ tr_modules(Conf) -> [{emqx_mod_acl_internal, [{acl_file, conf_get("acl.acl_file", Conf)}]}] ]). -broker__session_locking_strategy(mapping) -> "emqx.session_locking_strategy"; -broker__session_locking_strategy(type) -> session_locking_strategy(); -broker__session_locking_strategy(default) -> quorum; -broker__session_locking_strategy(_) -> undefined. - -%% @doc Shared Subscription Dispatch Strategy.randomly pick a subscriber -%% round robin alive subscribers one message after another -%% pick a random subscriber and stick to it -%% hash client ID to a group member -broker__shared_subscription_strategy(mapping) -> "emqx.shared_subscription_strategy"; -broker__shared_subscription_strategy(type) -> shared_subscription_strategy(); -broker__shared_subscription_strategy(default) -> round_robin; -broker__shared_subscription_strategy(_) -> undefined. - -%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. -%% Change this toggle only when there are many wildcard topics. -%% key: mnesia translational updates with per-key locks. recommended for single node setup. -%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. -%% global: global lock protected updates. recommended for larger cluster. -%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster -broker__perf__route_lock_type(mapping) -> "emqx.route_lock_type"; -broker__perf__route_lock_type(type) -> route_lock_type(); -broker__perf__route_lock_type(default) -> key; -broker__perf__route_lock_type(_) -> undefined. - tr_sysmon(Conf) -> Keys = maps:to_list(conf_get("sysmon", Conf, #{})), [{binary_to_atom(K), maps:get(value, V)} || {K, V} <- Keys]. tr_os_mon(Conf) -> [{cpu_check_interval, conf_get("os_mon.cpu_check_interval", Conf)} - , {cpu_high_watermark, conf_get("os_mon.cpu_high_watermark", Conf) * 100} - , {cpu_low_watermark, conf_get("os_mon.cpu_low_watermark", Conf) * 100} - , {mem_check_interval, conf_get("os_mon.mem_check_interval", Conf)} - , {sysmem_high_watermark, conf_get("os_mon.sysmem_high_watermark", Conf) * 100} - , {procmem_high_watermark, conf_get("os_mon.procmem_high_watermark", Conf) * 100} + , {cpu_high_watermark, conf_get("os_mon.cpu_high_watermark", Conf) * 100} + , {cpu_low_watermark, conf_get("os_mon.cpu_low_watermark", Conf) * 100} + , {mem_check_interval, conf_get("os_mon.mem_check_interval", Conf)} + , {sysmem_high_watermark, conf_get("os_mon.sysmem_high_watermark", Conf) * 100} + , {procmem_high_watermark, conf_get("os_mon.procmem_high_watermark", Conf) * 100} ]. tr_vm_mon(Conf) -> [ {check_interval, conf_get("vm_mon.check_interval", Conf)} - , {process_high_watermark, conf_get("vm_mon.process_high_watermark", Conf) * 100} - , {process_low_watermark, conf_get("vm_mon.process_low_watermark", Conf) * 100} + , {process_high_watermark, conf_get("vm_mon.process_high_watermark", Conf) * 100} + , {process_low_watermark, conf_get("vm_mon.process_low_watermark", Conf) * 100} ]. tr_alarm(Conf) -> - [ {actions, [list_to_atom(Action) || Action <- string:tokens(conf_get("alarm.actions", Conf), ",")]} - , {size_limit, conf_get("alarm.size_limit", Conf)} - , {validity_period, conf_get("alarm.validity_period", Conf)} + [ {actions, [list_to_atom(Action) || Action <- conf_get("alarm.actions", Conf)]} + , {size_limit, conf_get("alarm.size_limit", Conf)} + , {validity_period, conf_get("alarm.validity_period", Conf)} ]. tr_telemetry(Conf) -> [ {enabled, conf_get("telemetry.enabled", Conf)} - , {url, conf_get("telemetry.url", Conf)} - , {report_interval, conf_get("telemetry.report_interval", Conf)} + , {url, conf_get("telemetry.url", Conf)} + , {report_interval, conf_get("telemetry.report_interval", Conf)} ]. %% helpers options(static, Conf) -> - [{seeds, [list_to_atom(S) || S <- string:tokens(conf_get("cluster.static.seeds", Conf, ""), ",")]}]; + [{seeds, [list_to_atom(S) || S <- conf_get("cluster.static.seeds", Conf, "")]}]; options(mcast, Conf) -> {ok, Addr} = inet:parse_address(conf_get("cluster.mcast.addr", Conf)), {ok, Iface} = inet:parse_address(conf_get("cluster.mcast.iface", Conf)), - Ports = [list_to_integer(S) || S <- string:tokens(conf_get("cluster.mcast.ports", Conf), ",")], + Ports = [list_to_integer(S) || S <- conf_get("cluster.mcast.ports", Conf)], [{addr, Addr}, {ports, Ports}, {iface, Iface}, - {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, - {loop, conf_get("cluster.mcast.loop", Conf, true)}]; + {ttl, conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, conf_get("cluster.mcast.loop", Conf, true)}]; options(dns, Conf) -> [{name, conf_get("cluster.dns.name", Conf)}, - {app, conf_get("cluster.dns.app", Conf)}]; + {app, conf_get("cluster.dns.app", Conf)}]; options(etcd, Conf) -> Namespace = "cluster.etcd.ssl", SslOpts = fun(C) -> Options = keys(Namespace, C), lists:map(fun(Key) -> {list_to_atom(Key), conf_get([Namespace, Key], Conf)} end, Options) end, - [{server, string:tokens(conf_get("cluster.etcd.server", Conf), ",")}, - {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, - {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, - {ssl_options, SslOpts(Conf)}]; + [{server, conf_get("cluster.etcd.server", Conf)}, + {prefix, conf_get("cluster.etcd.prefix", Conf, "emqxcl")}, + {node_ttl, conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, filter(SslOpts(Conf))}]; options(k8s, Conf) -> [{apiserver, conf_get("cluster.k8s.apiserver", Conf)}, - {service_name, conf_get("cluster.k8s.service_name", Conf)}, - {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, - {app_name, conf_get("cluster.k8s.app_name", Conf)}, - {namespace, conf_get("cluster.k8s.namespace", Conf)}, - {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; + {service_name, conf_get("cluster.k8s.service_name", Conf)}, + {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, conf_get("cluster.k8s.app_name", Conf)}, + {namespace, conf_get("cluster.k8s.namespace", Conf)}, + {suffix, conf_get("cluster.k8s.suffix", Conf, "")}]; options(manual, _Conf) -> []. @@ -1223,15 +883,15 @@ burst_limit(["disabled"]) -> {false, {20000, 1000}}; burst_limit([Count, Window]) -> {true, {list_to_integer(Count), - case hocon_postprocess:duration(Window) of - Secs when is_integer(Secs) -> Secs; - _ -> error({duration, Window}) + case to_duration(Window) of + {ok, I} -> I; + {error, R} -> error({duration, R}) end}}. %% For creating additional log files for specific log levels. additional_log_files(Conf) -> LogLevel = ["debug", "info", "notice", "warning", - "error", "critical", "alert", "emergency"], + "error", "critical", "alert", "emergency"], additional_log_files(Conf, LogLevel, []). additional_log_files(_Conf, [], Acc) -> @@ -1242,44 +902,52 @@ additional_log_files(Conf, [L | More], Acc) -> F -> additional_log_files(Conf, More, [{L, F} | Acc]) end. -rate_limit(Val) -> - [L, D] = string:tokens(Val, ", "), - Limit = case cuttlefish_bytesize:parse(L) of - Sz when is_integer(Sz) -> Sz; - {error, Reason1} -> error(Reason1) +rate_limit_byte_dur([L, D]) -> + Limit = case to_bytesize(L) of + {ok, I0} -> I0; + {error, R0} -> error({bytesize, R0}) end, - Duration = case cuttlefish_duration:parse(D, s) of - Secs when is_integer(Secs) -> Secs; + Duration = case to_duration_s(D) of + {ok, I1} -> I1; + {error, R1} -> error({duration, R1}) + end, + {Limit, Duration}. + +rate_limit_num_dur([L, D]) -> + Limit = case string:to_integer(L) of + {Int, []} when is_integer(Int) -> Int; + _ -> error("failed to parse bytesize string") + end, + Duration = case to_duration_s(D) of + {ok, I} -> I; {error, Reason} -> error(Reason) end, {Limit, Duration}. map_zones(_, undefined) -> {undefined, undefined}; -map_zones("force_gc_policy", Val) -> - [Count, Bytes] = string:tokens(Val, "| "), - GcPolicy = case cuttlefish_bytesize:parse(Bytes) of +map_zones("force_gc_policy", [Count, Bytes]) -> + GcPolicy = case to_bytesize(Bytes) of {error, Reason} -> - error(Reason); - Bytes1 -> + error({bytesize, Reason}); + {ok, Bytes1} -> #{bytes => Bytes1, count => list_to_integer(Count)} end, {force_gc_policy, GcPolicy}; -map_zones("force_shutdown_policy", "default") -> +map_zones("force_shutdown_policy", ["default"]) -> WordSize = erlang:system_info(wordsize), {DefaultLen, DefaultSize} = case WordSize of 8 -> % arch_64 - {10000, cuttlefish_bytesize:parse("64MB")}; + {10000, hocon_postprocess:bytesize("64MB")}; 4 -> % arch_32 - {1000, cuttlefish_bytesize:parse("32MB")} + {1000, hocon_postprocess:bytesize("32MB")} end, {force_shutdown_policy, #{message_queue_len => DefaultLen, max_heap_size => DefaultSize div WordSize }}; -map_zones("force_shutdown_policy", Val) -> - [Len, Siz] = string:tokens(Val, "| "), +map_zones("force_shutdown_policy", [Len, Siz]) -> WordSize = erlang:system_info(wordsize), MaxSiz = case WordSize of 8 -> % arch_64 @@ -1288,19 +956,19 @@ map_zones("force_shutdown_policy", Val) -> (1 bsl 27) - 1 end, ShutdownPolicy = - case cuttlefish_bytesize:parse(Siz) of + case to_bytesize(Siz) of {error, Reason} -> error(Reason); - Siz1 when Siz1 > MaxSiz -> - cuttlefish:invalid(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); - Siz1 -> + {ok, Siz1} when Siz1 > MaxSiz -> + error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + {ok, Siz1} -> #{message_queue_len => list_to_integer(Len), max_heap_size => Siz1 div WordSize} end, {force_shutdown_policy, ShutdownPolicy}; map_zones("mqueue_priorities", Val) -> case Val of - "none" -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE + ["none"] -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE _ -> MqueuePriorities = lists:foldl(fun(T, Acc) -> %% NOTE: space in "= " is intended @@ -1308,7 +976,7 @@ map_zones("mqueue_priorities", Val) -> P = list_to_integer(Prio), (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), maps:put(iolist_to_binary(Topic), P, Acc) - end, #{}, string:tokens(Val, ",")), + end, #{}, Val), {mqueue_priorities, MqueuePriorities} end; map_zones("mountpoint", Val) -> @@ -1317,45 +985,45 @@ map_zones("response_information", Val) -> {response_information, iolist_to_binary(Val)}; map_zones("rate_limit", Conf) -> Messages = case conf_get("conn_messages_in", #{value => Conf}) of - undefined -> - []; - M -> - [{conn_messages_in, rate_limit(M)}] - end, + undefined -> + []; + M -> + [{conn_messages_in, rate_limit_num_dur(M)}] + end, Bytes = case conf_get("conn_bytes_in", #{value => Conf}) of - undefined -> - []; - B -> - [{conn_bytes_in, rate_limit(B)}] + undefined -> + []; + B -> + [{conn_bytes_in, rate_limit_byte_dur(B)}] end, {ratelimit, Messages ++ Bytes}; map_zones("conn_congestion", Conf) -> Alarm = case conf_get("alarm", #{value => Conf}) of + undefined -> + []; + A -> + [{conn_congestion_alarm_enabled, A}] + end, + MinAlarm = case conf_get("min_alarm_sustain_duration", #{value => Conf}) of undefined -> []; - A -> - [{conn_congestion_alarm_enabled, A}] + M -> + [{conn_congestion_min_alarm_sustain_duration, M}] end, - MinAlarm = case conf_get("min_alarm_sustain_duration", #{value => Conf}) of - undefined -> - []; - M -> - [{conn_congestion_min_alarm_sustain_duration, M}] - end, Alarm ++ MinAlarm; map_zones("quota", Conf) -> Conn = case conf_get("conn_messages_routing", #{value => Conf}) of - undefined -> - []; - C -> - [{conn_messages_routing, rate_limit(C)}] - end, + undefined -> + []; + C -> + [{conn_messages_routing, rate_limit_num_dur(C)}] + end, Overall = case conf_get("overall_messages_routing", #{value => Conf}) of - undefined -> - []; - O -> - [{overall_messages_routing, rate_limit(O)}] - end, + undefined -> + []; + O -> + [{overall_messages_routing, rate_limit_num_dur(O)}] + end, {quota, Conn ++ Overall}; map_zones(Opt, Val) -> {list_to_atom(Opt), Val}. @@ -1382,47 +1050,134 @@ conf_get(Key, Conf, Default) -> V end. +filter(Opts) -> + [{K, V} || {K, V} <- Opts, V =/= undefined]. + +%% generate a ssl field. +%% ssl("emqx", #{"verify" => verify_peer}) will return +%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} +%% , {"certfile", t(string(), "emqx.certfile", undefined)} +%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} +%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} +%% , {"server_name_indication", "emqx.server_name_indication", undefined)} +%% ... +ssl(Mapping, Defaults) -> + M = fun (Field) -> + case (Mapping) of + undefined -> undefined; + _ -> Mapping ++ "." ++ Field + end end, + D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, + [ {"enable", t(flag(), M("enable"), D("enable"))} + , {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))} + , {"certfile", t(string(), M("certfile"), D("certfile"))} + , {"keyfile", t(string(), M("keyfile"), D("keyfile"))} + , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} + , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} + , {"secure_renegotiate", t(flag(), M("secure_renegotiate"), D("secure_renegotiate"))} + , {"reuse_sessions", t(flag(), M("reuse_sessions"), D("reuse_sessions"))} + , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} + , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} + , {"depth", t(integer(), M("depth"), D("depth"))} + , {"password", t(string(), M("key_password"), D("key_password"))} + , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} + , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), + D("server_name_indication"))} + , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} + , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} + , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. + +tr_ssl(Field, Conf) -> + Versions = case conf_get([Field, "tls_versions"], Conf) of + undefined -> undefined; + Vs -> [list_to_existing_atom(V) || V <- Vs] + end, + TLSCiphers = conf_get([Field, "ciphers"], Conf), + PSKCiphers = conf_get([Field, "psk_ciphers"], Conf), + Ciphers = ciphers(TLSCiphers, PSKCiphers, Field), + case emqx_schema:conf_get([Field, "enable"], Conf) of + X when X =:= true orelse X =:= undefined -> + filter([{versions, Versions}, + {ciphers, Ciphers}, + {user_lookup_fun, user_lookup_fun(PSKCiphers)}, + {handshake_timeout, conf_get([Field, "handshake_timeout"], Conf)}, + {depth, conf_get([Field, "depth"], Conf)}, + {password, conf_get([Field, "key_password"], Conf)}, + {dhfile, conf_get([Field, "dhfile"], Conf)}, + {keyfile, emqx_schema:conf_get([Field, "keyfile"], Conf)}, + {certfile, emqx_schema:conf_get([Field, "certfile"], Conf)}, + {cacertfile, emqx_schema:conf_get([Field, "cacertfile"], Conf)}, + {verify, emqx_schema:conf_get([Field, "verify"], Conf)}, + {fail_if_no_peer_cert, conf_get([Field, "fail_if_no_peer_cert"], Conf)}, + {secure_renegotiate, conf_get([Field, "secure_renegotiate"], Conf)}, + {reuse_sessions, conf_get([Field, "reuse_sessions"], Conf)}, + {honor_cipher_order, conf_get([Field, "honor_cipher_order"], Conf)}, + {server_name_indication, emqx_schema:conf_get([Field, "server_name_indication"], Conf)} + ]); + _ -> + [] + end. + +map_psk_ciphers(PSKCiphers) -> + lists:map( + fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; + ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; + ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; + ("PSK-RC4-SHA") -> {psk, rc4_128, sha} + end, PSKCiphers). + +ciphers(undefined, undefined, _) -> + undefined; +ciphers(TLSCiphers, undefined, _) -> + TLSCiphers; +ciphers(undefined, PSKCiphers, _) -> + map_psk_ciphers(PSKCiphers); +ciphers(_, _, Field) -> + error(Field ++ ".ciphers and " ++ Field ++ ".psk_ciphers cannot be configured at the same time"). + +user_lookup_fun(undefined) -> + undefined; +user_lookup_fun(_PSKCiphers) -> + {fun emqx_psk:lookup/3, <<>>}. + +tr_password_hash(Field, Conf) -> + case emqx_schema:conf_get([Field, "password_hash"], Conf) of + [Hash] -> list_to_atom(Hash); + [Prefix, Suffix] -> {list_to_atom(Prefix), list_to_atom(Suffix)}; + [Hash, MacFun, Iterations, Dklen] -> {list_to_atom(Hash), list_to_atom(MacFun), + list_to_integer(Iterations), list_to_integer(Dklen)}; + _ -> plain + end. + + %% @private return a list of keys in a parent field -spec(keys(string(), hocon:config()) -> [string()]). keys(Parent, Conf) -> [binary_to_list(B) || B <- maps:keys(conf_get(Parent, Conf, #{}))]. +-spec ceiling(float()) -> integer(). +ceiling(X) -> + T = erlang:trunc(X), + case (X - T) of + Neg when Neg < 0 -> T; + Pos when Pos > 0 -> T + 1; + _ -> T + end. + %% types -duration(type) -> duration(); -duration(_) -> undefined. +t(T) -> + fun (type) -> T; (_) -> undefined end. -flag(type) -> flag(); -flag(_) -> undefined. +t(T, M, D) -> + fun (type) -> T; (mapping) -> M; (default) -> D; (_) -> undefined end. -string(type) -> string(); -string(_) -> undefined. - -integer(type) -> integer(); -integer(_) -> undefined. - -bytesize(type) -> bytesize(); -bytesize(_) -> undefined. - -boolean(type) -> boolean(); -boolean(_) -> undefined. - -duration(M, D) -> - fun (type) -> duration(); (mapping) -> M; (default) -> D; (_) -> undefined end. -duration_s(M, D) -> - fun (type) -> duration_s(); (mapping) -> M; (default) -> D; (_) -> undefined end. -flag(M, D) -> - fun (type) -> flag(); (mapping) -> M; (default) -> D; (_) -> undefined end. -string(M, D) -> - fun (type) -> string(); (mapping) -> M; (default) -> D; (_) -> undefined end. -integer(M, D) -> - fun (type) -> integer(); (mapping) -> M; (default) -> D; (_) -> undefined end. -bytesize(M, D) -> - fun (type) -> bytesize(); (mapping) -> M; (default) -> D; (_) -> undefined end. -boolean(M, D) -> - fun (type) -> boolean(); (mapping) -> M; (default) -> D; (_) -> undefined end. -percent(M, D) -> - fun (type) -> percent(); (mapping) -> M; (default) -> D; (_) -> undefined end. +t(T, M, D, O) -> + fun (type) -> T; + (mapping) -> M; + (default) -> D; + (override_env) -> O; + (_) -> undefined end. ref(Field) -> fun (type) -> Field; (_) -> undefined end. @@ -1431,13 +1186,38 @@ to_flag(Str) -> {ok, hocon_postprocess:onoff(Str)}. to_duration(Str) -> - {ok, hocon_postprocess:duration(Str)}. + case hocon_postprocess:duration(Str) of + I when is_integer(I) -> {ok, I}; + _ -> {error, Str} + end. to_duration_s(Str) -> - {ok, hocon_postprocess:duration(Str) div 1000}. + case hocon_postprocess:duration(Str) of + I when is_integer(I) -> {ok, ceiling(I / 1000)}; + _ -> {error, Str} + end. to_bytesize(Str) -> - {ok, hocon_postprocess:bytesize(Str)}. + case hocon_postprocess:bytesize(Str) of + I when is_integer(I) -> {ok, I}; + _ -> {error, Str} + end. to_percent(Str) -> {ok, hocon_postprocess:percent(Str)}. + +to_comma_separated_list(Str) -> + {ok, string:tokens(Str, ", ")}. + +to_bar_separated_list(Str) -> + {ok, string:tokens(Str, "| ")}. + +to_ip_port(Str) -> + case string:tokens(Str, ":") of + [Ip, Port] -> + case inet:parse_address(Ip) of + {ok, R} -> {ok, {R, list_to_integer(Port)}}; + _ -> {error, Str} + end; + _ -> {error, Str} + end. From a9efdaeacd0d5934b37e9a846a39755fcf6f1048 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Sat, 29 May 2021 20:44:39 +0900 Subject: [PATCH 49/58] fix(emqx_rule_engine_SUITE): use emqx_ct_helpers:read_schema_configs --- apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 2e82ef0cd..d9675cc6b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -2559,18 +2559,10 @@ start_apps() -> local_path("etc/emqx_rule_engine.conf")}]]. start_apps(App, SchemaFile, ConfigFile) -> - read_schema_configs(App, SchemaFile, ConfigFile), + emqx_ct_helpers:read_schema_configs(SchemaFile, ConfigFile), set_special_configs(App), {ok, _} = application:ensure_all_started(App). -read_schema_configs(App, SchemaFile, ConfigFile) -> - ct:pal("Read configs - SchemaFile: ~p, ConfigFile: ~p", [SchemaFile, ConfigFile]), - Schema = cuttlefish_schema:files([SchemaFile]), - {ok, Conf} = hocon:load(ConfigFile, #{format => proplists}), - NewConfig = cuttlefish_generator:map(Schema, Conf), - Vals = proplists:get_value(App, NewConfig, []), - [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. - deps_path(App, RelativePath) -> %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir %% but priv dir is From 47f7b35a602a309e9d78cdf3a4f0f617d370c74d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Sun, 30 May 2021 15:27:40 +0900 Subject: [PATCH 50/58] chore(bin/emqx): add todo on hocon --- bin/emqx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/emqx b/bin/emqx index 1487b8940..5f1b6d213 100755 --- a/bin/emqx +++ b/bin/emqx @@ -199,15 +199,15 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi + ## todo: include license conf option to hocon escript + ## EMQX_LICENSE_CONF_OPTION="" + ## if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then + ## EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" + ## fi set +e # shellcheck disable=SC2086 HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" - echo $HOCON_OUTPUT # shellcheck disable=SC2181 RESULT=$? set -e @@ -292,6 +292,7 @@ if [ -z "$NAME_ARG" ]; then NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')" else # for boot commands, inspect emqx.conf for node name + # todo: use get command from hocon escript NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \") fi fi @@ -318,7 +319,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" COOKIE="${EMQX_NODE_COOKIE:-}" if [ -z "$COOKIE" ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then - COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}') + COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \") else # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" From ad1ef732aea0e4f4978cca43e089e3a5c9ed619b Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 1 Jun 2021 14:39:12 +0900 Subject: [PATCH 51/58] chore(emqx_schema): suppress dialyzer warnings temporarily --- src/emqx_schema.erl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/emqx_schema.erl b/src/emqx_schema.erl index 5a0cc210d..bac2e5cfe 100644 --- a/src/emqx_schema.erl +++ b/src/emqx_schema.erl @@ -1,5 +1,12 @@ -module(emqx_schema). +% tmp +-dialyzer(no_return). +-dialyzer(no_match). +-dialyzer(no_contracts). +-dialyzer(no_unused). +-dialyzer(no_fail_call). + -include_lib("typerefl/include/types.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. From 9cb2facfe14afc8ae4606e5c989a60abd824879d Mon Sep 17 00:00:00 2001 From: z8674558 Date: Tue, 1 Jun 2021 21:05:04 +0900 Subject: [PATCH 52/58] chore(emqx_resource): unify dependency hocon to root rebar.config --- apps/emqx_resource/rebar.config | 3 +-- rebar.config | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 148af92bd..4e88043de 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -12,7 +12,6 @@ {dialyzer, [{warnings, [unmatched_returns, error_handling]} ]}. -{deps, [ {hocon, {git, "https://github.com/emqx/hocon", {branch, "master"}}} - , {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} +{deps, [ {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}} ]}. diff --git a/rebar.config b/rebar.config index f52f5cfba..0aeecb386 100644 --- a/rebar.config +++ b/rebar.config @@ -56,7 +56,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.2.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.3.0"}}} ]}. {xref_ignores, From 52e69b20b4dae1a480e47ec62cac3c710eac30d8 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Wed, 2 Jun 2021 13:39:50 +0900 Subject: [PATCH 53/58] chore(rebar.config): bump hocon; no more unused import --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 0aeecb386..c1e355d1d 100644 --- a/rebar.config +++ b/rebar.config @@ -7,7 +7,7 @@ %% rebar.config.rendered if environment DEBUG is set. {edoc_opts, [{preprocess,true}]}. -{erl_opts, [warn_unused_vars,warn_shadow_vars, +{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import, warn_obsolete_guard,compressed, {d, snk_kind, msg}]}. @@ -56,7 +56,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.3.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.3.2"}}} ]}. {xref_ignores, From f7bd206b9ba91403db51a03a634122c89f877e0a Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 2 Jun 2021 14:18:00 +0200 Subject: [PATCH 54/58] chore: pin cuttlefish 4.0.1 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index c1e355d1d..d70dc4f82 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.9.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.0"}}} + , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} % TODO: delete when all apps moved to hocon , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} From 03519d7e61817acbc279e08194858d32fff80831 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 3 Jun 2021 10:55:37 +0800 Subject: [PATCH 55/58] fix(emqx_resource): HTTP APIs for emqx_resource not working --- apps/emqx_resource/include/emqx_resource.hrl | 2 +- apps/emqx_resource/rebar.config | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 3 +- apps/emqx_resource/src/emqx_resource.erl | 54 ++++++++++--- apps/emqx_resource/src/emqx_resource_api.erl | 17 ++-- .../src/emqx_resource_instance.erl | 78 ++++++++++++------- .../src/emqx_resource_transform.erl | 16 ++-- 7 files changed, 114 insertions(+), 58 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 1f75b453e..123854bc9 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -type resource_type() :: module(). -type instance_id() :: binary(). --type resource_config() :: jsx:json_term(). +-type resource_config() :: term(). -type resource_spec() :: map(). -type resource_state() :: term(). -type resource_data() :: #{ diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 4e88043de..66271ee56 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [ debug_info , nowarn_unused_import - %, {d, 'RESOURCE_DEBUG'} + , {d, 'RESOURCE_DEBUG'} ]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}. diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index af9f48cc6..13330b061 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -7,7 +7,8 @@ [kernel, stdlib, gproc, - hocon + hocon, + jsx ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b9107d328..c432e683e 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -65,13 +65,17 @@ , call_health_check/3 %% verify if the resource is working normally , call_stop/3 %% stop the instance , call_config_merge/4 %% merge the config when updating + , call_jsonify/2 + , call_api_reply_format/2 ]). -export([ list_instances/0 %% list all the instances, id only. , list_instances_verbose/0 %% list all the instances , get_instance/1 %% return the data of the instance , get_instance_by_type/1 %% return all the instances of the same resource type - , load_instances/1 %% load instances from config files + , load_instances_from_dir/1 %% load instances from a directory + , load_instance_from_file/1 %% load an instance from a config file + , load_instance_from_config/1 %% load an instance from a map or json-string config % , dependents/1 % , inc_counter/2 %% increment the counter of the instance % , inc_counter/3 %% increment the counter by a given integer @@ -81,14 +85,17 @@ -optional_callbacks([ on_query/4 , on_health_check/2 - , on_api_reply_format/1 , on_config_merge/3 + , on_jsonify/1 + , on_api_reply_format/1 ]). --callback on_api_reply_format(resource_data()) -> map(). +-callback on_api_reply_format(resource_data()) -> jsx:json_term(). -callback on_config_merge(resource_config(), resource_config(), term()) -> resource_config(). +-callback on_jsonify(resource_config()) -> jsx:json_term(). + %% when calling emqx_resource:start/1 -callback on_start(instance_id(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. @@ -208,9 +215,17 @@ list_instances_verbose() -> get_instance_by_type(ResourceType) -> emqx_resource_instance:lookup_by_type(ResourceType). --spec load_instances(Dir :: string()) -> ok. -load_instances(Dir) -> - emqx_resource_instance:load(Dir). +-spec load_instances_from_dir(Dir :: string()) -> ok. +load_instances_from_dir(Dir) -> + emqx_resource_instance:load_dir(Dir). + +-spec load_instance_from_file(File :: string()) -> ok. +load_instance_from_file(File) -> + emqx_resource_instance:load_file(File). + +-spec load_instance_from_config(binary() | map()) -> ok. +load_instance_from_config(Config) -> + emqx_resource_instance:load_config(Config). -spec call_start(instance_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. @@ -229,7 +244,28 @@ call_stop(InstId, Mod, ResourceState) -> -spec call_config_merge(module(), resource_config(), resource_config(), term()) -> resource_config(). call_config_merge(Mod, OldConfig, NewConfig, Params) -> - ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)). + case erlang:function_exported(Mod, on_jsonify, 1) of + true -> + ?SAFE_CALL(Mod:on_config_merge(OldConfig, NewConfig, Params)); + false when is_map(OldConfig), is_map(NewConfig) -> + maps:merge(OldConfig, NewConfig); + false -> + NewConfig + end. + +-spec call_jsonify(module(), resource_config()) -> jsx:json_term(). +call_jsonify(Mod, Config) -> + case erlang:function_exported(Mod, on_jsonify, 1) of + false -> Config; + true -> ?SAFE_CALL(Mod:on_jsonify(Config)) + end. + +-spec call_api_reply_format(module(), resource_data()) -> jsx:json_term(). +call_api_reply_format(Mod, Data) -> + case erlang:function_exported(Mod, on_api_reply_format, 1) of + false -> emqx_resource_api:default_api_reply_format(Data); + true -> ?SAFE_CALL(Mod:on_api_reply_format(Data)) + end. -spec parse_config(resource_type(), binary() | term()) -> {ok, resource_config()} | {error, term()}. @@ -240,7 +276,7 @@ parse_config(ResourceType, RawConfig) when is_binary(RawConfig) -> Error -> Error end; parse_config(ResourceType, RawConfigTerm) -> - parse_config(ResourceType, jsx:encode(#{<<"config">> => RawConfigTerm})). + parse_config(ResourceType, jsx:encode(#{config => RawConfigTerm})). -spec do_parse_config(resource_type(), map()) -> {ok, resource_config()} | {error, term()}. do_parse_config(ResourceType, MapConfig) -> @@ -261,7 +297,7 @@ resource_type_from_str(ResourceType) -> false -> {error, {invalid_resource, Mod}} end catch error:badarg -> - {error, {not_found, ResourceType}} + {error, {resource_not_found, ResourceType}} end. call_instance(InstId, Query) -> diff --git a/apps/emqx_resource/src/emqx_resource_api.erl b/apps/emqx_resource/src/emqx_resource_api.erl index cc32d8a11..1e19cdede 100644 --- a/apps/emqx_resource/src/emqx_resource_api.erl +++ b/apps/emqx_resource/src/emqx_resource_api.erl @@ -20,6 +20,9 @@ , put/3 , delete/3 ]). + +-export([default_api_reply_format/1]). + get_all(Mod, _Binding, _Params) -> {200, #{code => 0, data => [format_data(Mod, Data) || Data <- emqx_resource:list_instances_verbose()]}}. @@ -34,7 +37,7 @@ get(Mod, #{id := Id}, _Params) -> put(Mod, #{id := Id}, Params) -> ConfigParams = proplists:get_value(<<"config">>, Params), - ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params), + ResourceTypeStr = proplists:get_value(<<"resource_type">>, Params, #{}), case emqx_resource:resource_type_from_str(ResourceTypeStr) of {ok, ResourceType} -> do_put(Mod, stringnify(Id), ConfigParams, ResourceType, Params); @@ -63,15 +66,11 @@ delete(_Mod, #{id := Id}, _Params) -> end. format_data(Mod, Data) -> - case erlang:function_exported(Mod, on_api_reply_format, 1) of - false -> - default_api_reply_format(Data); - true -> - Mod:on_api_reply_format(Data) - end. + emqx_resource:call_api_reply_format(Mod, Data). -default_api_reply_format(#{id := Id, status := Status, config := Config}) -> - #{node => node(), id => Id, status => Status, config => Config}. +default_api_reply_format(#{id := Id, mod := Mod, status := Status, config := Config}) -> + #{node => node(), id => Id, status => Status, resource_type => Mod, + config => emqx_resource:call_jsonify(Mod, Config)}. stringnify(Bin) when is_binary(Bin) -> Bin; stringnify(Str) when is_list(Str) -> list_to_binary(Str); diff --git a/apps/emqx_resource/src/emqx_resource_instance.erl b/apps/emqx_resource/src/emqx_resource_instance.erl index ca5e4829e..dbbf052a1 100644 --- a/apps/emqx_resource/src/emqx_resource_instance.erl +++ b/apps/emqx_resource/src/emqx_resource_instance.erl @@ -23,10 +23,13 @@ -export([start_link/2]). %% load resource instances from *.conf files --export([ load/1 +-export([ load_dir/1 + , load_file/1 + , load_config/1 , lookup/1 , list_all/0 , lookup_by_type/1 + , create_local/3 ]). -export([ hash_call/2 @@ -82,8 +85,8 @@ lookup_by_type(ResourceType) -> [Data || #{mod := Mod} = Data <- list_all() , Mod =:= ResourceType]. --spec load(Dir :: string()) -> ok. -load(Dir) -> +-spec load_dir(Dir :: string()) -> ok. +load_dir(Dir) -> lists:foreach(fun load_file/1, filelib:wildcard(filename:join([Dir, "*.conf"]))). load_file(File) -> @@ -91,40 +94,51 @@ load_file(File) -> {error, Reason} -> logger:error("load resource from ~p failed: ~p", [File, Reason]); RawConfig -> - case hocon:binary(RawConfig, #{format => map}) of - {ok, #{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr, - <<"config">> := MapConfig}} -> - case emqx_resource:resource_type_from_str(ResourceTypeStr) of - {ok, ResourceType} -> - parse_and_load_config(Id, ResourceType, MapConfig); - {error, Reason} -> - logger:error("no such resource type: ~s, ~p", - [ResourceTypeStr, Reason]) - end; + case load_config(RawConfig) of + {ok, Data} -> + logger:debug("loaded resource instance from file: ~p, data: ~p", + [File, Data]); {error, Reason} -> logger:error("load resource from ~p failed: ~p", [File, Reason]) end end. -parse_and_load_config(InstId, ResourceType, MapConfig) -> - case emqx_resource:parse_config(ResourceType, MapConfig) of - {error, Reason} -> - logger:error("parse config for resource ~p of type ~p failed: ~p", - [InstId, ResourceType, Reason]); - {ok, InstConf} -> - create_instance_local(InstId, ResourceType, InstConf) +-spec load_config(binary() | map()) -> {ok, resource_data()} | {error, term()}. +load_config(RawConfig) when is_binary(RawConfig) -> + case hocon:binary(RawConfig, #{format => map}) of + {ok, ConfigTerm} -> load_config(ConfigTerm); + Error -> Error + end; + +load_config(#{<<"id">> := Id, <<"resource_type">> := ResourceTypeStr} = Config) -> + MapConfig = maps:get(<<"config">>, Config, #{}), + case emqx_resource:resource_type_from_str(ResourceTypeStr) of + {ok, ResourceType} -> parse_and_load_config(Id, ResourceType, MapConfig); + Error -> Error end. -create_instance_local(InstId, ResourceType, InstConf) -> - case do_create(InstId, ResourceType, InstConf) of - {ok, Data} -> - logger:debug("created ~p resource instance: ~p from config: ~p, Data: ~p", - [ResourceType, InstId, InstConf, Data]); - {error, Reason} -> - logger:error("create ~p resource instance: ~p failed: ~p, config: ~p", - [ResourceType, InstId, Reason, InstConf]) +parse_and_load_config(InstId, ResourceType, MapConfig) -> + case emqx_resource:parse_config(ResourceType, MapConfig) of + {ok, InstConf} -> create_local(InstId, ResourceType, InstConf); + Error -> Error end. +create_local(InstId, ResourceType, InstConf) -> + case hash_call(InstId, {create, InstId, ResourceType, InstConf}, 15000) of + {ok, Data} -> {ok, Data}; + Error -> Error + end. + +save_config_to_disk(InstId, ResourceType, Config) -> + %% TODO: send an event to the config handler, and the hander (single process) + %% will dump configs for all instances (from an ETS table) to a file. + file:write_file(filename:join([emqx_data_dir(), binary_to_list(InstId) ++ ".conf"]), + jsx:encode(#{id => InstId, resource_type => ResourceType, + config => emqx_resource:call_jsonify(ResourceType, Config)})). + +emqx_data_dir() -> + "data". + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ @@ -205,7 +219,13 @@ do_create(InstId, ResourceType, Config) -> #{mod => ResourceType, config => Config, state => ResourceState, status => stopped}}), _ = do_health_check(InstId), - {ok, force_lookup(InstId)}; + case save_config_to_disk(InstId, ResourceType, Config) of + ok -> {ok, force_lookup(InstId)}; + {error, Reason} -> + logger:error("save config for ~p resource ~p to disk failed: ~p", + [ResourceType, InstId, Reason]), + {error, Reason} + end; {error, Reason} -> logger:error("start ~s resource ~s failed: ~p", [ResourceType, InstId, Reason]), {error, Reason} diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index 23e14013c..cd6c7e4ae 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -57,7 +57,8 @@ forms(_, []) -> []. form(Mod, Form) -> case Form of ?Q("-emqx_resource_api_path('@Path').") -> - {fix_spec_attrs() ++ fix_api_attrs(erl_syntax:concrete(Path)) ++ fix_api_exports(), + {fix_spec_attrs() ++ fix_api_attrs(Mod, erl_syntax:concrete(Path)) + ++ fix_api_exports(), [], fix_spec_funcs(Mod) ++ fix_api_funcs(Mod)}; _ -> @@ -75,18 +76,17 @@ fix_spec_funcs(_Mod) -> , ?Q("structs() -> [\"config\"].") ]. -fix_api_attrs(Path0) -> - BaseName = filename:basename(Path0), - Path = "/" ++ BaseName, +fix_api_attrs(Mod, Path) -> + BaseName = atom_to_list(Mod), [erl_syntax:revert( erl_syntax:attribute(?Q("rest_api"), [ erl_syntax:abstract(#{ - name => list_to_atom(Name ++ "_log_tracers"), + name => list_to_atom(Act ++ "_" ++ BaseName), method => Method, path => mk_path(Path, WithId), func => Func, - descr => Name ++ " the " ++ BaseName})])) - || {Name, Method, WithId, Func} <- [ + descr => Act ++ " the " ++ BaseName})])) + || {Act, Method, WithId, Func} <- [ {"list", 'GET', noid, api_get_all}, {"get", 'GET', id, api_get}, {"update", 'PUT', id, api_put}, @@ -110,5 +110,5 @@ fix_api_funcs(Mod) -> emqx_resource_api:delete('@Mod@', Binding, Params).")) ]. -mk_path(Path, id) -> Path ++ "/:bin:id"; +mk_path(Path, id) -> string:trim(Path, trailing, "/") ++ "/:bin:id"; mk_path(Path, noid) -> Path. From 9617b65d52f37d8877014c286a1b555f3fba60f4 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 3 Jun 2021 11:27:06 +0800 Subject: [PATCH 56/58] fix(dialyzer): incorrect function spec --- apps/emqx_resource/rebar.config | 2 +- apps/emqx_resource/src/emqx_resource.erl | 2 +- apps/emqx_resource/src/emqx_resource_transform.erl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index 66271ee56..4e88043de 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [ debug_info , nowarn_unused_import - , {d, 'RESOURCE_DEBUG'} + %, {d, 'RESOURCE_DEBUG'} ]}. {erl_first_files, ["src/emqx_resource_transform.erl"]}. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index c432e683e..7470b823d 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -223,7 +223,7 @@ load_instances_from_dir(Dir) -> load_instance_from_file(File) -> emqx_resource_instance:load_file(File). --spec load_instance_from_config(binary() | map()) -> ok. +-spec load_instance_from_config(binary() | map()) -> {ok, resource_data()} | {error, term()}. load_instance_from_config(Config) -> emqx_resource_instance:load_config(Config). diff --git a/apps/emqx_resource/src/emqx_resource_transform.erl b/apps/emqx_resource/src/emqx_resource_transform.erl index cd6c7e4ae..41e79bde7 100644 --- a/apps/emqx_resource/src/emqx_resource_transform.erl +++ b/apps/emqx_resource/src/emqx_resource_transform.erl @@ -29,7 +29,7 @@ parse_transform(Forms, _Opts) -> debug_print(Mod, Ts) -> {ok, Io} = file:open("./" ++ atom_to_list(Mod) ++ ".trans.erl", [write]), - do_debug_print(Io, Ts), + _ = do_debug_print(Io, Ts), file:close(Io). do_debug_print(Io, Ts) when is_list(Ts) -> From b4eb0c4df6fae0c4987617cafce8719b5b5d5e85 Mon Sep 17 00:00:00 2001 From: z8674558 Date: Thu, 3 Jun 2021 16:24:35 +0900 Subject: [PATCH 57/58] chore(bin/emqx): use hocon get --- bin/emqx | 4 ++-- rebar.config | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/emqx b/bin/emqx index 5f1b6d213..413501a04 100755 --- a/bin/emqx +++ b/bin/emqx @@ -293,7 +293,7 @@ if [ -z "$NAME_ARG" ]; then else # for boot commands, inspect emqx.conf for node name # todo: use get command from hocon escript - NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \") + NODENAME="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")" fi fi if [ -z "$NODENAME" ]; then @@ -319,7 +319,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}" COOKIE="${EMQX_NODE_COOKIE:-}" if [ -z "$COOKIE" ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then - COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \") + COOKIE="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")" else # shellcheck disable=SC2012,SC2086 LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)" diff --git a/rebar.config b/rebar.config index d70dc4f82..cf2a46aac 100644 --- a/rebar.config +++ b/rebar.config @@ -56,7 +56,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.3.2"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.4.0"}}} ]}. {xref_ignores, From 5795bcca6ac54fdcf4c6960158c2cca84875a3a0 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 4 Jun 2021 19:21:38 +0200 Subject: [PATCH 58/58] chore: fix app versions and check script --- .../emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- .../src/emqx_auth_mongo.app.src | 2 +- .../src/emqx_auth_mysql.app.src | 2 +- .../src/emqx_auth_pgsql.app.src | 2 +- .../src/emqx_auth_redis.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- .../src/emqx_management.app.src | 2 +- apps/emqx_recon/src/emqx_recon.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- scripts/apps-version-check.sh | 47 ++++++++++++++----- 11 files changed, 45 insertions(+), 22 deletions(-) diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index 8635c4834..1b76c32c8 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index cc4e72ef3..ab0b4ff56 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 221061f03..8a0d116cc 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index f70612262..e97487e21 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 6a38af2ce..419131566 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 52ce96a54..9841f5bcb 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.0"}, %% strict semver + {vsn, "4.4.0"}, %% strict semver {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 43fb5ba53..fe68fef44 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_recon/src/emqx_recon.app.src b/apps/emqx_recon/src/emqx_recon.app.src index 061ed8e93..e25e2bbc5 100644 --- a/apps/emqx_recon/src/emqx_recon.app.src +++ b/apps/emqx_recon/src/emqx_recon.app.src @@ -1,6 +1,6 @@ {application, emqx_recon, [{description, "EMQ X Recon Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,recon]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index b03abdac1..2e66734ec 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src index 92bd59a7a..1604198dc 100644 --- a/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src +++ b/lib-ce/emqx_dashboard/src/emqx_dashboard.app.src @@ -1,6 +1,6 @@ {application, emqx_dashboard, [{description, "EMQ X Web Dashboard"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.4.0"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel,stdlib,mnesia,minirest]}, diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 8de85cd8c..0134bf187 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -6,22 +6,45 @@ latest_release=$(git describe --tags "$(git rev-list --tags --max-count=1 --remo bad_app_count=0 -while read -r app; do - if [ "$app" != "emqx" ]; then - app_path="$app" +get_vsn() { + commit="$1" + app_src_file="$2" + if [ "$commit" = 'HEAD' ]; then + if [ -f "$app_src_file" ]; then + grep vsn "$app_src_file" | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true + fi else - app_path="." + git show "$commit":"$app_src_file" 2>/dev/null | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"' || true fi - src_file="$app_path/src/$(basename "$app").app.src" - old_app_version="$(git show "$latest_release":"$src_file" | grep vsn | grep -oE '"[0-9]+.[0-9]+.[0-9]+"' | tr -d '"')" - now_app_version=$(grep -E 'vsn' "$src_file" | grep -oE '"[0-9]+\.[0-9]+\.[0-9]+"' | tr -d '"') - if [ "$old_app_version" = "$now_app_version" ]; then - changed="$(git diff --name-only "$latest_release"...HEAD \ +} + +while read -r app_path; do + app=$(basename "$app_path") + src_file="$app_path/src/$app.app.src" + old_app_version="$(get_vsn "$latest_release" "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$old_app_version" = '' ]; then + old_app_version="$(get_vsn "$latest_release" 'src/emqx.app.src')" + fi + now_app_version="$(get_vsn 'HEAD' "$src_file")" + ## TODO: delete it after new version is released with emqx app in apps dir + if [ "$app" = 'emqx' ] && [ "$now_app_version" = '' ]; then + now_app_version="$(get_vsn 'HEAD' 'src/emqx.app.src')" + fi + if [ -z "$now_app_version" ]; then + echo "failed_to_get_new_app_vsn for $app" + exit 1 + fi + if [ -z "${old_app_version:-}" ]; then + echo "skiped checking new app ${app}" + elif [ "$old_app_version" = "$now_app_version" ]; then + lines="$(git diff --name-only "$latest_release"...HEAD \ -- "$app_path/src" \ -- "$app_path/priv" \ - -- "$app_path/c_src" | wc -l)" - if [ "$changed" -gt 0 ]; then - echo "$src_file needs a vsn bump" + -- "$app_path/c_src")" + if [ "$lines" != '' ]; then + echo "$src_file needs a vsn bump (old=$old_app_version)" + echo "changed: $lines" bad_app_count=$(( bad_app_count + 1)) fi fi